PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120
 121     # AMD CPUs
 122     athlon => 'AuthenticAMD',
 123     phenom  => 'AuthenticAMD',
 124     Opteron_G1  => 'AuthenticAMD',
 125     Opteron_G2  => 'AuthenticAMD',
 126     Opteron_G3  => 'AuthenticAMD',
 127     Opteron_G4  => 'AuthenticAMD',
 128     Opteron_G5  => 'AuthenticAMD',
 129
 130     # generic types, use vendor from host node
 131     host => 'default',
 132     kvm32 => 'default',
 133     kvm64 => 'default',
 134     qemu32 => 'default',
 135     qemu64 => 'default',
 136 };
 137
 138 my $cpu_fmt = {
 139     cputype => {
 140         description => "Emulated CPU type.",
 141         type => 'string',
 142         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 143         format_description => 'cputype',
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 218         minimum => 0,
 219         maximum => 500000,
 220         default => 1000,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012
 285 l24;; Linux 2.4 Kernel
 286 l26;; Linux 2.6/3.X Kernel
 287 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 288 EODESC
 289     },
 290     boot => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 294         pattern => '[acdn]{1,4}',
 295         default => 'cdn',
 296     },
 297     bootdisk => {
 298         optional => 1,
 299         type => 'string', format => 'pve-qm-bootdisk',
 300         description => "Enable booting from specified disk.",
 301         pattern => '(ide|sata|scsi|virtio)\d+',
 302     },
 303     smp => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPUs. Please use option -sockets instead.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     sockets => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPU sockets.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     cores => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of cores per socket.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     numa => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable NUMA.",
 328         default => 0,
 329     },
 330     hugepages => {
 331         optional => 1,
 332         type => 'string',
 333         description => "Enable/disable hugepages memory.",
 334         enum => [qw(any 2 1024)],
 335     },
 336     vcpus => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Number of hotplugged vcpus.",
 340         minimum => 1,
 341         default => 0,
 342     },
 343     acpi => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable ACPI.",
 347         default => 1,
 348     },
 349     agent => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable Qemu GuestAgent.",
 353         default => 0,
 354     },
 355     kvm => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable KVM hardware virtualization.",
 359         default => 1,
 360     },
 361     tdf => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable time drift fix.",
 365         default => 0,
 366     },
 367     localtime => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 371     },
 372     freeze => {
 373         optional => 1,
 374         type => 'boolean',
 375         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 376     },
 377     vga => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Select the VGA type.",
 381         verbose_description => "Select the VGA type. If you want to use high resolution" .
 382             " modes (>= 1280x1024x16) then you should use the options " .
 383             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 384             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 385             "display sever. For win* OS you can select how many independent " .
 386             "displays you want, Linux guests can add displays them self. " .
 387             "You can also run without any graphic card, using a serial device" .
 388             " as terminal.",
 389         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 390     },
 391     watchdog => {
 392         optional => 1,
 393         type => 'string', format => 'pve-qm-watchdog',
 394         description => "Create a virtual hardware watchdog device.",
 395         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 396             " (by a guest action), the watchdog must be periodically polled " .
 397             "by an agent inside the guest or else the watchdog will reset " .
 398             "the guest (or execute the respective action specified)",
 399     },
 400     startdate => {
 401         optional => 1,
 402         type => 'string',
 403         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 404         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 405         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 406         default => 'now',
 407     },
 408     startup =>  get_standard_option('pve-startup-order'),
 409     template => {
 410         optional => 1,
 411         type => 'boolean',
 412         description => "Enable/disable Template.",
 413         default => 0,
 414     },
 415     args => {
 416         optional => 1,
 417         type => 'string',
 418         description => "Arbitrary arguments passed to kvm.",
 419         verbose_description => <<EODESCR,
 420 Arbitrary arguments passed to kvm, for example:
 421
 422 args: -no-reboot -no-hpet
 423
 424 NOTE: this option is for experts only.
 425 EODESCR
 426     },
 427     tablet => {
 428         optional => 1,
 429         type => 'boolean',
 430         default => 1,
 431         description => "Enable/disable the USB tablet device.",
 432         verbose_description => "Enable/disable the USB tablet device. This device is " .
 433             "usually needed to allow absolute mouse positioning with VNC. " .
 434             "Else the mouse runs out of sync with normal VNC clients. " .
 435             "If you're running lots of console-only guests on one host, " .
 436             "you may consider disabling this to save some context switches. " .
 437             "This is turned off by default if you use spice (-vga=qxl).",
 438     },
 439     migrate_speed => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 443         minimum => 0,
 444         default => 0,
 445     },
 446     migrate_downtime => {
 447         optional => 1,
 448         type => 'number',
 449         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 450         minimum => 0,
 451         default => 0.1,
 452     },
 453     cdrom => {
 454         optional => 1,
 455         type => 'string', format => 'pve-qm-ide',
 456         typetext => 'volume',
 457         description => "This is an alias for option -ide2",
 458     },
 459     cpu => {
 460         optional => 1,
 461         description => "Emulated CPU type.",
 462         type => 'string',
 463         format => $cpu_fmt,
 464     },
 465     parent => get_standard_option('pve-snapshot-name', {
 466         optional => 1,
 467         description => "Parent snapshot name. This is used internally, and should not be modified.",
 468     }),
 469     snaptime => {
 470         optional => 1,
 471         description => "Timestamp for snapshots.",
 472         type => 'integer',
 473         minimum => 0,
 474     },
 475     vmstate => {
 476         optional => 1,
 477         type => 'string', format => 'pve-volume-id',
 478         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 479     },
 480     machine => {
 481         description => "Specific the Qemu machine type.",
 482         type => 'string',
 483         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 484         maxLength => 40,
 485         optional => 1,
 486     },
 487     smbios1 => {
 488         description => "Specify SMBIOS type 1 fields.",
 489         type => 'string', format => 'pve-qm-smbios1',
 490         maxLength => 256,
 491         optional => 1,
 492     },
 493     protection => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 497         default => 0,
 498     },
 499     bios => {
 500         optional => 1,
 501         type => 'string',
 502         enum => [ qw(seabios ovmf) ],
 503         description => "Select BIOS implementation.",
 504         default => 'seabios',
 505     },
 506 };
 507
 508 # what about other qemu settings ?
 509 #cpu => 'string',
 510 #machine => 'string',
 511 #fda => 'file',
 512 #fdb => 'file',
 513 #mtdblock => 'file',
 514 #sd => 'file',
 515 #pflash => 'file',
 516 #snapshot => 'bool',
 517 #bootp => 'file',
 518 ##tftp => 'dir',
 519 ##smb => 'dir',
 520 #kernel => 'file',
 521 #append => 'string',
 522 #initrd => 'file',
 523 ##soundhw => 'string',
 524
 525 while (my ($k, $v) = each %$confdesc) {
 526     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 527 }
 528
 529 my $MAX_IDE_DISKS = 4;
 530 my $MAX_SCSI_DISKS = 14;
 531 my $MAX_VIRTIO_DISKS = 16;
 532 my $MAX_SATA_DISKS = 6;
 533 my $MAX_USB_DEVICES = 5;
 534 my $MAX_NETS = 32;
 535 my $MAX_UNUSED_DISKS = 8;
 536 my $MAX_HOSTPCI_DEVICES = 4;
 537 my $MAX_SERIAL_PORTS = 4;
 538 my $MAX_PARALLEL_PORTS = 3;
 539 my $MAX_NUMA = 8;
 540
 541 my $numa_fmt = {
 542     cpus => {
 543         type => "string",
 544         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 545         description => "CPUs accessing this NUMA node.",
 546         format_description => "id[-id];...",
 547     },
 548     memory => {
 549         type => "number",
 550         description => "Amount of memory this NUMA node provides.",
 551         optional => 1,
 552     },
 553     hostnodes => {
 554         type => "string",
 555         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 556         description => "Host NUMA nodes to use.",
 557         format_description => "id[-id];...",
 558         optional => 1,
 559     },
 560     policy => {
 561         type => 'string',
 562         enum => [qw(preferred bind interleave)],
 563         description => "NUMA allocation policy.",
 564         optional => 1,
 565     },
 566 };
 567 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 568 my $numadesc = {
 569     optional => 1,
 570     type => 'string', format => $numa_fmt,
 571     description => "NUMA topology.",
 572 };
 573 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 574
 575 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 576     $confdesc->{"numa$i"} = $numadesc;
 577 }
 578
 579 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 580                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 581                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 582 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 583
 584 my $net_fmt_bridge_descr = <<__EOD__;
 585 Bridge to attach the network device to. The Proxmox VE standard bridge
 586 is called 'vmbr0'.
 587
 588 If you do not specify a bridge, we create a kvm user (NATed) network
 589 device, which provides DHCP and DNS services. The following addresses
 590 are used:
 591
 592  10.0.2.2   Gateway
 593  10.0.2.3   DNS Server
 594  10.0.2.4   SMB Server
 595
 596 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 597 __EOD__
 598
 599 my $net_fmt = {
 600     macaddr => {
 601         type => 'string',
 602         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 603         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 604         format_description => "XX:XX:XX:XX:XX:XX",
 605         optional => 1,
 606     },
 607     model => {
 608         type => 'string',
 609         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 610         format_description => 'model',
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => {
 738         type => 'string',
 739         format_description => 'image format',
 740         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 741         description => "The drive's backing file's data format.",
 742         optional => 1,
 743     },
 744     size => {
 745         type => 'string',
 746         format => 'disk-size',
 747         format_description => 'DiskSize',
 748         description => "Disk size. This is purely informational and has no effect.",
 749         optional => 1,
 750     },
 751     backup => {
 752         type => 'boolean',
 753         description => "Whether the drive should be included when making backups.",
 754         optional => 1,
 755     },
 756     werror => {
 757         type => 'string',
 758         enum => [qw(enospc ignore report stop)],
 759         description => 'Write error action.',
 760         optional => 1,
 761     },
 762     aio => {
 763         type => 'string',
 764         enum => [qw(native threads)],
 765         description => 'AIO type to use.',
 766         optional => 1,
 767     },
 768     discard => {
 769         type => 'string',
 770         enum => [qw(ignore on)],
 771         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 772         optional => 1,
 773     },
 774     detect_zeroes => {
 775         type => 'boolean',
 776         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 777         optional => 1,
 778     },
 779     serial => {
 780         type => 'string',
 781         format => 'urlencoded',
 782         format_description => 'serial',
 783         maxLength => 20*3, # *3 since it's %xx url enoded
 784         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 785         optional => 1,
 786     }
 787 );
 788
 789 my %rerror_fmt = (
 790     rerror => {
 791         type => 'string',
 792         enum => [qw(ignore report stop)],
 793         description => 'Read error action.',
 794         optional => 1,
 795     },
 796 );
 797
 798 my %iothread_fmt = ( iothread => {
 799         type => 'boolean',
 800         description => "Whether to use iothreads for this drive",
 801         optional => 1,
 802 });
 803
 804 my %model_fmt = (
 805     model => {
 806         type => 'string',
 807         format => 'urlencoded',
 808         format_description => 'model',
 809         maxLength => 40*3, # *3 since it's %xx url enoded
 810         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 811         optional => 1,
 812     },
 813 );
 814
 815 my %queues_fmt = (
 816     queues => {
 817         type => 'integer',
 818         description => "Number of queues.",
 819         minimum => 2,
 820         optional => 1
 821     }
 822 );
 823
 824 my $add_throttle_desc = sub {
 825     my ($key, $type, $what, $unit, $longunit) = @_;
 826     $drivedesc_base{$key} = {
 827         type => $type,
 828         format_description => $unit,
 829         description => "Maximum $what speed in $longunit per second.",
 830         optional => 1,
 831     };
 832 };
 833 # throughput: (leaky bucket)
 834 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 835 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 836 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 837 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 838 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 839 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 840 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 841 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 842 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 843
 844 # pools: (pool of IO before throttling starts taking effect)
 845 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 846 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 847 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 848 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 849 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 850 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 851
 852 my $ide_fmt = {
 853     %drivedesc_base,
 854     %rerror_fmt,
 855     %model_fmt,
 856 };
 857 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 858
 859 my $idedesc = {
 860     optional => 1,
 861     type => 'string', format => $ide_fmt,
 862     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 865
 866 my $scsi_fmt = {
 867     %drivedesc_base,
 868     %iothread_fmt,
 869     %queues_fmt,
 870 };
 871 my $scsidesc = {
 872     optional => 1,
 873     type => 'string', format => $scsi_fmt,
 874     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 875 };
 876 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 877
 878 my $sata_fmt = {
 879     %drivedesc_base,
 880     %rerror_fmt,
 881 };
 882 my $satadesc = {
 883     optional => 1,
 884     type => 'string', format => $sata_fmt,
 885     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 886 };
 887 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 888
 889 my $virtio_fmt = {
 890     %drivedesc_base,
 891     %iothread_fmt,
 892     %rerror_fmt,
 893 };
 894 my $virtiodesc = {
 895     optional => 1,
 896     type => 'string', format => $virtio_fmt,
 897     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 898 };
 899 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 900
 901 my $alldrive_fmt = {
 902     %drivedesc_base,
 903     %rerror_fmt,
 904     %iothread_fmt,
 905     %model_fmt,
 906     %queues_fmt,
 907 };
 908
 909 my $efidisk_fmt = {
 910     volume => { alias => 'file' },
 911     file => {
 912         type => 'string',
 913         format => 'pve-volume-id-or-qm-path',
 914         default_key => 1,
 915         format_description => 'volume',
 916         description => "The drive's backing volume.",
 917     },
 918     format => {
 919         type => 'string',
 920         format_description => 'image format',
 921         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 922         description => "The drive's backing file's data format.",
 923         optional => 1,
 924     },
 925     size => {
 926         type => 'string',
 927         format => 'disk-size',
 928         format_description => 'DiskSize',
 929         description => "Disk size. This is purely informational and has no effect.",
 930         optional => 1,
 931     },
 932 };
 933
 934 my $efidisk_desc = {
 935     optional => 1,
 936     type => 'string', format => $efidisk_fmt,
 937     description => "Configure a Disk for storing EFI vars",
 938 };
 939
 940 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 941
 942 my $usb_fmt = {
 943     host => {
 944         default_key => 1,
 945         type => 'string', format => 'pve-qm-usb-device',
 946         format_description => 'HOSTUSBDEVICE|spice',
 947         description => <<EODESCR,
 948 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 949
 950  'bus-port(.port)*' (decimal numbers) or
 951  'vendor_id:product_id' (hexadeciaml numbers) or
 952  'spice'
 953
 954 You can use the 'lsusb -t' command to list existing usb devices.
 955
 956 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 957
 958 The value 'spice' can be used to add a usb redirection devices for spice.
 959 EODESCR
 960     },
 961     usb3 => {
 962         optional => 1,
 963         type => 'boolean',
 964         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 965         default => 0,
 966     },
 967 };
 968
 969 my $usbdesc = {
 970     optional => 1,
 971     type => 'string', format => $usb_fmt,
 972     description => "Configure an USB device (n is 0 to 4).",
 973 };
 974 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 975
 976 # NOTE: the match-groups of this regex are used in parse_hostpci
 977 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 978 my $hostpci_fmt = {
 979     host => {
 980         default_key => 1,
 981         type => 'string',
 982         pattern => qr/$PCIRE(;$PCIRE)*/,
 983         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 984         description => <<EODESCR,
 985 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 986 of PCI virtual functions of the host. HOSTPCIID syntax is:
 987
 988 'bus:dev.func' (hexadecimal numbers)
 989
 990 You can us the 'lspci' command to list existing PCI devices.
 991 EODESCR
 992     },
 993     rombar => {
 994         type => 'boolean',
 995         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 996         optional => 1,
 997         default => 1,
 998     },
 999     pcie => {
1000         type => 'boolean',
1001         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1002         optional => 1,
1003         default => 0,
1004     },
1005     'x-vga' => {
1006         type => 'boolean',
1007         description =>  "Enable vfio-vga device support.",
1008         optional => 1,
1009         default => 0,
1010     },
1011 };
1012 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1013
1014 my $hostpcidesc = {
1015         optional => 1,
1016         type => 'string', format => 'pve-qm-hostpci',
1017         description => "Map host PCI devices into guest.",
1018         verbose_description =>  <<EODESCR,
1019 Map host PCI devices into guest.
1020
1021 NOTE: This option allows direct access to host hardware. So it is no longer
1022 possible to migrate such machines - use with special care.
1023
1024 CAUTION: Experimental! User reported problems with this option.
1025 EODESCR
1026 };
1027 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1028
1029 my $serialdesc = {
1030         optional => 1,
1031         type => 'string',
1032         pattern => '(/dev/.+|socket)',
1033         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1034         verbose_description =>  <<EODESCR,
1035 Create a serial device inside the VM (n is 0 to 3), and pass through a
1036 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1037 host side (use 'qm terminal' to open a terminal connection).
1038
1039 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1040
1041 CAUTION: Experimental! User reported problems with this option.
1042 EODESCR
1043 };
1044
1045 my $paralleldesc= {
1046         optional => 1,
1047         type => 'string',
1048         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1049         description =>  "Map host parallel devices (n is 0 to 2).",
1050         verbose_description =>  <<EODESCR,
1051 Map host parallel devices (n is 0 to 2).
1052
1053 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1054
1055 CAUTION: Experimental! User reported problems with this option.
1056 EODESCR
1057 };
1058
1059 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1060     $confdesc->{"parallel$i"} = $paralleldesc;
1061 }
1062
1063 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1064     $confdesc->{"serial$i"} = $serialdesc;
1065 }
1066
1067 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1068     $confdesc->{"hostpci$i"} = $hostpcidesc;
1069 }
1070
1071 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1072     $drivename_hash->{"ide$i"} = 1;
1073     $confdesc->{"ide$i"} = $idedesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1077     $drivename_hash->{"sata$i"} = 1;
1078     $confdesc->{"sata$i"} = $satadesc;
1079 }
1080
1081 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1082     $drivename_hash->{"scsi$i"} = 1;
1083     $confdesc->{"scsi$i"} = $scsidesc ;
1084 }
1085
1086 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1087     $drivename_hash->{"virtio$i"} = 1;
1088     $confdesc->{"virtio$i"} = $virtiodesc;
1089 }
1090
1091 $drivename_hash->{efidisk0} = 1;
1092 $confdesc->{efidisk0} = $efidisk_desc;
1093
1094 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1095     $confdesc->{"usb$i"} = $usbdesc;
1096 }
1097
1098 my $unuseddesc = {
1099     optional => 1,
1100     type => 'string', format => 'pve-volume-id',
1101     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1102 };
1103
1104 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1105     $confdesc->{"unused$i"} = $unuseddesc;
1106 }
1107
1108 my $kvm_api_version = 0;
1109
1110 sub kvm_version {
1111
1112     return $kvm_api_version if $kvm_api_version;
1113
1114     my $fh = IO::File->new("</dev/kvm") ||
1115         return 0;
1116
1117     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1118         $kvm_api_version = $v;
1119     }
1120
1121     $fh->close();
1122
1123     return  $kvm_api_version;
1124 }
1125
1126 my $kvm_user_version;
1127
1128 sub kvm_user_version {
1129
1130     return $kvm_user_version if $kvm_user_version;
1131
1132     $kvm_user_version = 'unknown';
1133
1134     my $code = sub {
1135         my $line = shift;
1136         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1137             $kvm_user_version = $2;
1138         }
1139     };
1140
1141     eval { run_command("kvm -version", outfunc => $code); };
1142     warn $@ if $@;
1143
1144     return $kvm_user_version;
1145
1146 }
1147
1148 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1149
1150 sub valid_drive_names {
1151     # order is important - used to autoselect boot disk
1152     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1153             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1154             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1155             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1156             'efidisk0');
1157 }
1158
1159 sub is_valid_drivename {
1160     my $dev = shift;
1161
1162     return defined($drivename_hash->{$dev});
1163 }
1164
1165 sub option_exists {
1166     my $key = shift;
1167     return defined($confdesc->{$key});
1168 }
1169
1170 sub nic_models {
1171     return $nic_model_list;
1172 }
1173
1174 sub os_list_description {
1175
1176     return {
1177         other => 'Other',
1178         wxp => 'Windows XP',
1179         w2k => 'Windows 2000',
1180         w2k3 =>, 'Windows 2003',
1181         w2k8 => 'Windows 2008',
1182         wvista => 'Windows Vista',
1183         win7 => 'Windows 7',
1184         win8 => 'Windows 8/2012',
1185         l24 => 'Linux 2.4',
1186         l26 => 'Linux 2.6',
1187     };
1188 }
1189
1190 my $cdrom_path;
1191
1192 sub get_cdrom_path {
1193
1194     return  $cdrom_path if $cdrom_path;
1195
1196     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1197     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1198     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1199 }
1200
1201 sub get_iso_path {
1202     my ($storecfg, $vmid, $cdrom) = @_;
1203
1204     if ($cdrom eq 'cdrom') {
1205         return get_cdrom_path();
1206     } elsif ($cdrom eq 'none') {
1207         return '';
1208     } elsif ($cdrom =~ m|^/|) {
1209         return $cdrom;
1210     } else {
1211         return PVE::Storage::path($storecfg, $cdrom);
1212     }
1213 }
1214
1215 # try to convert old style file names to volume IDs
1216 sub filename_to_volume_id {
1217     my ($vmid, $file, $media) = @_;
1218
1219     if (!($file eq 'none' || $file eq 'cdrom' ||
1220           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1221
1222         return undef if $file =~ m|/|;
1223
1224         if ($media && $media eq 'cdrom') {
1225             $file = "local:iso/$file";
1226         } else {
1227             $file = "local:$vmid/$file";
1228         }
1229     }
1230
1231     return $file;
1232 }
1233
1234 sub verify_media_type {
1235     my ($opt, $vtype, $media) = @_;
1236
1237     return if !$media;
1238
1239     my $etype;
1240     if ($media eq 'disk') {
1241         $etype = 'images';
1242     } elsif ($media eq 'cdrom') {
1243         $etype = 'iso';
1244     } else {
1245         die "internal error";
1246     }
1247
1248     return if ($vtype eq $etype);
1249
1250     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1251 }
1252
1253 sub cleanup_drive_path {
1254     my ($opt, $storecfg, $drive) = @_;
1255
1256     # try to convert filesystem paths to volume IDs
1257
1258     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1259         ($drive->{file} !~ m|^/dev/.+|) &&
1260         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1261         ($drive->{file} !~ m/^\d+$/)) {
1262         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1263         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1264         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1265         verify_media_type($opt, $vtype, $drive->{media});
1266         $drive->{file} = $volid;
1267     }
1268
1269     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1270 }
1271
1272 sub parse_hotplug_features {
1273     my ($data) = @_;
1274
1275     my $res = {};
1276
1277     return $res if $data eq '0';
1278
1279     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1280
1281     foreach my $feature (PVE::Tools::split_list($data)) {
1282         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1283             $res->{$1} = 1;
1284         } else {
1285             die "invalid hotplug feature '$feature'\n";
1286         }
1287     }
1288     return $res;
1289 }
1290
1291 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1292 sub pve_verify_hotplug_features {
1293     my ($value, $noerr) = @_;
1294
1295     return $value if parse_hotplug_features($value);
1296
1297     return undef if $noerr;
1298
1299     die "unable to parse hotplug option\n";
1300 }
1301
1302 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1303 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1304 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1305 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1306 #        [,iothread=on][,serial=serial][,model=model]
1307
1308 sub parse_drive {
1309     my ($key, $data) = @_;
1310
1311     my ($interface, $index);
1312
1313     if ($key =~ m/^([^\d]+)(\d+)$/) {
1314         $interface = $1;
1315         $index = $2;
1316     } else {
1317         return undef;
1318     }
1319
1320     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1321                                      : $confdesc->{$key}->{format};
1322     if (!$desc) {
1323         warn "invalid drive key: $key\n";
1324         return undef;
1325     }
1326     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1327     return undef if !$res;
1328     $res->{interface} = $interface;
1329     $res->{index} = $index;
1330
1331     my $error = 0;
1332     foreach my $opt (qw(bps bps_rd bps_wr)) {
1333         if (my $bps = defined(delete $res->{$opt})) {
1334             if (defined($res->{"m$opt"})) {
1335                 warn "both $opt and m$opt specified\n";
1336                 ++$error;
1337                 next;
1338             }
1339             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1340         }
1341     }
1342     return undef if $error;
1343
1344     return undef if $res->{mbps_rd} && $res->{mbps};
1345     return undef if $res->{mbps_wr} && $res->{mbps};
1346     return undef if $res->{iops_rd} && $res->{iops};
1347     return undef if $res->{iops_wr} && $res->{iops};
1348
1349     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1350         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1351         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1352         return undef if $res->{interface} eq 'virtio';
1353     }
1354
1355     if (my $size = $res->{size}) {
1356         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1357     }
1358
1359     return $res;
1360 }
1361
1362 sub print_drive {
1363     my ($vmid, $drive) = @_;
1364     my $data = { %$drive };
1365     delete $data->{$_} for qw(index interface);
1366     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1367 }
1368
1369 sub scsi_inquiry {
1370     my($fh, $noerr) = @_;
1371
1372     my $SG_IO = 0x2285;
1373     my $SG_GET_VERSION_NUM = 0x2282;
1374
1375     my $versionbuf = "\x00" x 8;
1376     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1377     if (!$ret) {
1378         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1379         return undef;
1380     }
1381     my $version = unpack("I", $versionbuf);
1382     if ($version < 30000) {
1383         die "scsi generic interface too old\n"  if !$noerr;
1384         return undef;
1385     }
1386
1387     my $buf = "\x00" x 36;
1388     my $sensebuf = "\x00" x 8;
1389     my $cmd = pack("C x3 C x1", 0x12, 36);
1390
1391     # see /usr/include/scsi/sg.h
1392     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1393
1394     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1395                       length($sensebuf), 0, length($buf), $buf,
1396                       $cmd, $sensebuf, 6000);
1397
1398     $ret = ioctl($fh, $SG_IO, $packet);
1399     if (!$ret) {
1400         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1401         return undef;
1402     }
1403
1404     my @res = unpack($sg_io_hdr_t, $packet);
1405     if ($res[17] || $res[18]) {
1406         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1407         return undef;
1408     }
1409
1410     my $res = {};
1411     (my $byte0, my $byte1, $res->{vendor},
1412      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1413
1414     $res->{removable} = $byte1 & 128 ? 1 : 0;
1415     $res->{type} = $byte0 & 31;
1416
1417     return $res;
1418 }
1419
1420 sub path_is_scsi {
1421     my ($path) = @_;
1422
1423     my $fh = IO::File->new("+<$path") || return undef;
1424     my $res = scsi_inquiry($fh, 1);
1425     close($fh);
1426
1427     return $res;
1428 }
1429
1430 sub machine_type_is_q35 {
1431     my ($conf) = @_;
1432
1433     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1434 }
1435
1436 sub print_tabletdevice_full {
1437     my ($conf) = @_;
1438
1439     my $q35 = machine_type_is_q35($conf);
1440
1441     # we use uhci for old VMs because tablet driver was buggy in older qemu
1442     my $usbbus = $q35 ? "ehci" : "uhci";
1443
1444     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1445 }
1446
1447 sub print_drivedevice_full {
1448     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1449
1450     my $device = '';
1451     my $maxdev = 0;
1452
1453     if ($drive->{interface} eq 'virtio') {
1454         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1455         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1456         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1457     } elsif ($drive->{interface} eq 'scsi') {
1458
1459         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1460         my $unit = $drive->{index} % $maxdev;
1461         my $devicetype = 'hd';
1462         my $path = '';
1463         if (drive_is_cdrom($drive)) {
1464             $devicetype = 'cd';
1465         } else {
1466             if ($drive->{file} =~ m|^/|) {
1467                 $path = $drive->{file};
1468                 if (my $info = path_is_scsi($path)) {
1469                     if ($info->{type} == 0) {
1470                         $devicetype = 'block';
1471                     } elsif ($info->{type} == 1) { # tape
1472                         $devicetype = 'generic';
1473                     }
1474                 }
1475             } else {
1476                  $path = PVE::Storage::path($storecfg, $drive->{file});
1477             }
1478
1479             if($path =~ m/^iscsi\:\/\//){
1480                 $devicetype = 'generic';
1481             }
1482         }
1483
1484         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1485             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1486         } else {
1487             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1488         }
1489
1490     } elsif ($drive->{interface} eq 'ide'){
1491         $maxdev = 2;
1492         my $controller = int($drive->{index} / $maxdev);
1493         my $unit = $drive->{index} % $maxdev;
1494         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1495
1496         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1497         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1498             $model = URI::Escape::uri_unescape($model);
1499             $device .= ",model=$model";
1500         }
1501     } elsif ($drive->{interface} eq 'sata'){
1502         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1503         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1504         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1505     } elsif ($drive->{interface} eq 'usb') {
1506         die "implement me";
1507         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1508     } else {
1509         die "unsupported interface type";
1510     }
1511
1512     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1513
1514     return $device;
1515 }
1516
1517 sub get_initiator_name {
1518     my $initiator;
1519
1520     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1521     while (defined(my $line = <$fh>)) {
1522         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1523         $initiator = $1;
1524         last;
1525     }
1526     $fh->close();
1527
1528     return $initiator;
1529 }
1530
1531 sub print_drive_full {
1532     my ($storecfg, $vmid, $drive) = @_;
1533
1534     my $path;
1535     my $volid = $drive->{file};
1536     my $format;
1537
1538     if (drive_is_cdrom($drive)) {
1539         $path = get_iso_path($storecfg, $vmid, $volid);
1540     } else {
1541         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1542         if ($storeid) {
1543             $path = PVE::Storage::path($storecfg, $volid);
1544             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1545             $format = qemu_img_format($scfg, $volname);
1546         } else {
1547             $path = $volid;
1548             $format = "raw";
1549         }
1550    }
1551
1552     my $opts = '';
1553     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1554     foreach my $o (@qemu_drive_options) {
1555         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1556     }
1557     if (my $serial = $drive->{serial}) {
1558         $serial = URI::Escape::uri_unescape($serial);
1559         $opts .= ",serial=$serial";
1560     }
1561
1562     $opts .= ",format=$format" if $format && !$drive->{format};
1563
1564     foreach my $o (qw(bps bps_rd bps_wr)) {
1565         my $v = $drive->{"m$o"};
1566         $opts .= ",$o=" . int($v*1024*1024) if $v;
1567     }
1568
1569     my $cache_direct = 0;
1570
1571     if (my $cache = $drive->{cache}) {
1572         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1573     } elsif (!drive_is_cdrom($drive)) {
1574         $opts .= ",cache=none";
1575         $cache_direct = 1;
1576     }
1577
1578     # aio native works only with O_DIRECT
1579     if (!$drive->{aio}) {
1580         if($cache_direct) {
1581             $opts .= ",aio=native";
1582         } else {
1583             $opts .= ",aio=threads";
1584         }
1585     }
1586
1587     if (!drive_is_cdrom($drive)) {
1588         my $detectzeroes;
1589         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1590             $detectzeroes = 'off';
1591         } elsif ($drive->{discard}) {
1592             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1593         } else {
1594             # This used to be our default with discard not being specified:
1595             $detectzeroes = 'on';
1596         }
1597         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1598     }
1599
1600     my $pathinfo = $path ? "file=$path," : '';
1601
1602     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1603 }
1604
1605 sub print_netdevice_full {
1606     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1607
1608     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1609
1610     my $device = $net->{model};
1611     if ($net->{model} eq 'virtio') {
1612          $device = 'virtio-net-pci';
1613      };
1614
1615     my $pciaddr = print_pci_addr("$netid", $bridges);
1616     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1617     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1618         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1619         my $vectors = $net->{queues} * 2 + 2;
1620         $tmpstr .= ",vectors=$vectors,mq=on";
1621     }
1622     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1623
1624     if ($use_old_bios_files) {
1625         my $romfile;
1626         if ($device eq 'virtio-net-pci') {
1627             $romfile = 'pxe-virtio.rom';
1628         } elsif ($device eq 'e1000') {
1629             $romfile = 'pxe-e1000.rom';
1630         } elsif ($device eq 'ne2k') {
1631             $romfile = 'pxe-ne2k_pci.rom';
1632         } elsif ($device eq 'pcnet') {
1633             $romfile = 'pxe-pcnet.rom';
1634         } elsif ($device eq 'rtl8139') {
1635             $romfile = 'pxe-rtl8139.rom';
1636         }
1637         $tmpstr .= ",romfile=$romfile" if $romfile;
1638     }
1639
1640     return $tmpstr;
1641 }
1642
1643 sub print_netdev_full {
1644     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1645
1646     my $i = '';
1647     if ($netid =~ m/^net(\d+)$/) {
1648         $i = int($1);
1649     }
1650
1651     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1652
1653     my $ifname = "tap${vmid}i$i";
1654
1655     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1656     die "interface name '$ifname' is too long (max 15 character)\n"
1657         if length($ifname) >= 16;
1658
1659     my $vhostparam = '';
1660     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1661
1662     my $vmname = $conf->{name} || "vm$vmid";
1663
1664     my $netdev = "";
1665     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1666
1667     if ($net->{bridge}) {
1668         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1669     } else {
1670         $netdev = "type=user,id=$netid,hostname=$vmname";
1671     }
1672
1673     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1674
1675     return $netdev;
1676 }
1677
1678
1679 sub print_cpu_device {
1680     my ($conf, $id) = @_;
1681
1682     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1683     my $cpu = $nokvm ? "qemu64" : "kvm64";
1684     if (my $cputype = $conf->{cpu}) {
1685         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1686             or die "Cannot parse cpu description: $cputype\n";
1687         $cpu = $cpuconf->{cputype};
1688     }
1689
1690     my $sockets = 1;
1691     $sockets = $conf->{sockets} if  $conf->{sockets};
1692     my $cores = $conf->{cores} || 1;
1693
1694     my $current_core = ($id - 1) % $cores;
1695     my $current_socket = int(($id - $current_core)/$cores);
1696
1697     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1698 }
1699
1700 sub drive_is_cdrom {
1701     my ($drive) = @_;
1702
1703     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1704
1705 }
1706
1707 sub parse_number_sets {
1708     my ($set) = @_;
1709     my $res = [];
1710     foreach my $part (split(/;/, $set)) {
1711         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1712             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1713             push @$res, [ $1, $2 ];
1714         } else {
1715             die "invalid range: $part\n";
1716         }
1717     }
1718     return $res;
1719 }
1720
1721 sub parse_numa {
1722     my ($data) = @_;
1723
1724     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1725     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1726     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1727     return $res;
1728 }
1729
1730 sub parse_hostpci {
1731     my ($value) = @_;
1732
1733     return undef if !$value;
1734
1735     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1736
1737     my @idlist = split(/;/, $res->{host});
1738     delete $res->{host};
1739     foreach my $id (@idlist) {
1740         if ($id =~ /^$PCIRE$/) {
1741             if (defined($2)) {
1742                 push @{$res->{pciid}}, { id => $1, function => $2 };
1743             } else {
1744                 my $pcidevices = lspci($1);
1745                 $res->{pciid} = $pcidevices->{$1};
1746             }
1747         } else {
1748             # should have been caught by parse_property_string already
1749             die "failed to parse PCI id: $id\n";
1750         }
1751     }
1752     return $res;
1753 }
1754
1755 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1756 sub parse_net {
1757     my ($data) = @_;
1758
1759     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1760     if ($@) {
1761         warn $@;
1762         return undef;
1763     }
1764     if (!defined($res->{macaddr})) {
1765         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1766         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1767     }
1768     return $res;
1769 }
1770
1771 sub print_net {
1772     my $net = shift;
1773
1774     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1775 }
1776
1777 sub add_random_macs {
1778     my ($settings) = @_;
1779
1780     foreach my $opt (keys %$settings) {
1781         next if $opt !~ m/^net(\d+)$/;
1782         my $net = parse_net($settings->{$opt});
1783         next if !$net;
1784         $settings->{$opt} = print_net($net);
1785     }
1786 }
1787
1788 sub vm_is_volid_owner {
1789     my ($storecfg, $vmid, $volid) = @_;
1790
1791     if ($volid !~  m|^/|) {
1792         my ($path, $owner);
1793         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1794         if ($owner && ($owner == $vmid)) {
1795             return 1;
1796         }
1797     }
1798
1799     return undef;
1800 }
1801
1802 sub split_flagged_list {
1803     my $text = shift || '';
1804     $text =~ s/[,;]/ /g;
1805     $text =~ s/^\s+//;
1806     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1807 }
1808
1809 sub join_flagged_list {
1810     my ($how, $lst) = @_;
1811     join $how, map { $lst->{$_} . $_ } keys %$lst;
1812 }
1813
1814 sub vmconfig_delete_pending_option {
1815     my ($conf, $key, $force) = @_;
1816
1817     delete $conf->{pending}->{$key};
1818     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1819     $pending_delete_hash->{$key} = $force ? '!' : '';
1820     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1821 }
1822
1823 sub vmconfig_undelete_pending_option {
1824     my ($conf, $key) = @_;
1825
1826     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1827     delete $pending_delete_hash->{$key};
1828
1829     if (%$pending_delete_hash) {
1830         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1831     } else {
1832         delete $conf->{pending}->{delete};
1833     }
1834 }
1835
1836 sub vmconfig_register_unused_drive {
1837     my ($storecfg, $vmid, $conf, $drive) = @_;
1838
1839     if (!drive_is_cdrom($drive)) {
1840         my $volid = $drive->{file};
1841         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1842             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1843         }
1844     }
1845 }
1846
1847 sub vmconfig_cleanup_pending {
1848     my ($conf) = @_;
1849
1850     # remove pending changes when nothing changed
1851     my $changes;
1852     foreach my $opt (keys %{$conf->{pending}}) {
1853         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1854             $changes = 1;
1855             delete $conf->{pending}->{$opt};
1856         }
1857     }
1858
1859     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1860     my $pending_delete_hash = {};
1861     while (my ($opt, $force) = each %$current_delete_hash) {
1862         if (defined($conf->{$opt})) {
1863             $pending_delete_hash->{$opt} = $force;
1864         } else {
1865             $changes = 1;
1866         }
1867     }
1868
1869     if (%$pending_delete_hash) {
1870         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1871     } else {
1872         delete $conf->{pending}->{delete};
1873     }
1874
1875     return $changes;
1876 }
1877
1878 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1879 my $smbios1_fmt = {
1880     uuid => {
1881         type => 'string',
1882         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1883         format_description => 'UUID',
1884         description => "Set SMBIOS1 UUID.",
1885         optional => 1,
1886     },
1887     version => {
1888         type => 'string',
1889         pattern => '\S+',
1890         format_description => 'string',
1891         description => "Set SMBIOS1 version.",
1892         optional => 1,
1893     },
1894     serial => {
1895         type => 'string',
1896         pattern => '\S+',
1897         format_description => 'string',
1898         description => "Set SMBIOS1 serial number.",
1899         optional => 1,
1900     },
1901     manufacturer => {
1902         type => 'string',
1903         pattern => '\S+',
1904         format_description => 'string',
1905         description => "Set SMBIOS1 manufacturer.",
1906         optional => 1,
1907     },
1908     product => {
1909         type => 'string',
1910         pattern => '\S+',
1911         format_description => 'string',
1912         description => "Set SMBIOS1 product ID.",
1913         optional => 1,
1914     },
1915     sku => {
1916         type => 'string',
1917         pattern => '\S+',
1918         format_description => 'string',
1919         description => "Set SMBIOS1 SKU string.",
1920         optional => 1,
1921     },
1922     family => {
1923         type => 'string',
1924         pattern => '\S+',
1925         format_description => 'string',
1926         description => "Set SMBIOS1 family string.",
1927         optional => 1,
1928     },
1929 };
1930
1931 sub parse_smbios1 {
1932     my ($data) = @_;
1933
1934     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1935     warn $@ if $@;
1936     return $res;
1937 }
1938
1939 sub print_smbios1 {
1940     my ($smbios1) = @_;
1941     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1942 }
1943
1944 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1945
1946 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1947 sub verify_bootdisk {
1948     my ($value, $noerr) = @_;
1949
1950     return $value if is_valid_drivename($value);
1951
1952     return undef if $noerr;
1953
1954     die "invalid boot disk '$value'\n";
1955 }
1956
1957 sub parse_watchdog {
1958     my ($value) = @_;
1959
1960     return undef if !$value;
1961
1962     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1963     warn $@ if $@;
1964     return $res;
1965 }
1966
1967 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1968 sub verify_usb_device {
1969     my ($value, $noerr) = @_;
1970
1971     return $value if parse_usb_device($value);
1972
1973     return undef if $noerr;
1974
1975     die "unable to parse usb device\n";
1976 }
1977
1978 # add JSON properties for create and set function
1979 sub json_config_properties {
1980     my $prop = shift;
1981
1982     foreach my $opt (keys %$confdesc) {
1983         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1984         $prop->{$opt} = $confdesc->{$opt};
1985     }
1986
1987     return $prop;
1988 }
1989
1990 sub check_type {
1991     my ($key, $value) = @_;
1992
1993     die "unknown setting '$key'\n" if !$confdesc->{$key};
1994
1995     my $type = $confdesc->{$key}->{type};
1996
1997     if (!defined($value)) {
1998         die "got undefined value\n";
1999     }
2000
2001     if ($value =~ m/[\n\r]/) {
2002         die "property contains a line feed\n";
2003     }
2004
2005     if ($type eq 'boolean') {
2006         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2007         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2008         die "type check ('boolean') failed - got '$value'\n";
2009     } elsif ($type eq 'integer') {
2010         return int($1) if $value =~ m/^(\d+)$/;
2011         die "type check ('integer') failed - got '$value'\n";
2012     } elsif ($type eq 'number') {
2013         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2014         die "type check ('number') failed - got '$value'\n";
2015     } elsif ($type eq 'string') {
2016         if (my $fmt = $confdesc->{$key}->{format}) {
2017             PVE::JSONSchema::check_format($fmt, $value);
2018             return $value;
2019         }
2020         $value =~ s/^\"(.*)\"$/$1/;
2021         return $value;
2022     } else {
2023         die "internal error"
2024     }
2025 }
2026
2027 sub check_iommu_support{
2028     #fixme : need to check IOMMU support
2029     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2030
2031     my $iommu=1;
2032     return $iommu;
2033
2034 }
2035
2036 sub touch_config {
2037     my ($vmid) = @_;
2038
2039     my $conf = PVE::QemuConfig->config_file($vmid);
2040     utime undef, undef, $conf;
2041 }
2042
2043 sub destroy_vm {
2044     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2045
2046     my $conffile = PVE::QemuConfig->config_file($vmid);
2047
2048     my $conf = PVE::QemuConfig->load_config($vmid);
2049
2050     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2051
2052     # only remove disks owned by this VM
2053     foreach_drive($conf, sub {
2054         my ($ds, $drive) = @_;
2055
2056         return if drive_is_cdrom($drive);
2057
2058         my $volid = $drive->{file};
2059
2060         return if !$volid || $volid =~ m|^/|;
2061
2062         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2063         return if !$path || !$owner || ($owner != $vmid);
2064
2065         PVE::Storage::vdisk_free($storecfg, $volid);
2066     });
2067
2068     if ($keep_empty_config) {
2069         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2070     } else {
2071         unlink $conffile;
2072     }
2073
2074     # also remove unused disk
2075     eval {
2076         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2077
2078         eval {
2079             PVE::Storage::foreach_volid($dl, sub {
2080                 my ($volid, $sid, $volname, $d) = @_;
2081                 PVE::Storage::vdisk_free($storecfg, $volid);
2082             });
2083         };
2084         warn $@ if $@;
2085
2086     };
2087     warn $@ if $@;
2088 }
2089
2090 sub parse_vm_config {
2091     my ($filename, $raw) = @_;
2092
2093     return undef if !defined($raw);
2094
2095     my $res = {
2096         digest => Digest::SHA::sha1_hex($raw),
2097         snapshots => {},
2098         pending => {},
2099     };
2100
2101     $filename =~ m|/qemu-server/(\d+)\.conf$|
2102         || die "got strange filename '$filename'";
2103
2104     my $vmid = $1;
2105
2106     my $conf = $res;
2107     my $descr;
2108     my $section = '';
2109
2110     my @lines = split(/\n/, $raw);
2111     foreach my $line (@lines) {
2112         next if $line =~ m/^\s*$/;
2113
2114         if ($line =~ m/^\[PENDING\]\s*$/i) {
2115             $section = 'pending';
2116             if (defined($descr)) {
2117                 $descr =~ s/\s+$//;
2118                 $conf->{description} = $descr;
2119             }
2120             $descr = undef;
2121             $conf = $res->{$section} = {};
2122             next;
2123
2124         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2125             $section = $1;
2126             if (defined($descr)) {
2127                 $descr =~ s/\s+$//;
2128                 $conf->{description} = $descr;
2129             }
2130             $descr = undef;
2131             $conf = $res->{snapshots}->{$section} = {};
2132             next;
2133         }
2134
2135         if ($line =~ m/^\#(.*)\s*$/) {
2136             $descr = '' if !defined($descr);
2137             $descr .= PVE::Tools::decode_text($1) . "\n";
2138             next;
2139         }
2140
2141         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2142             $descr = '' if !defined($descr);
2143             $descr .= PVE::Tools::decode_text($2);
2144         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2145             $conf->{snapstate} = $1;
2146         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2147             my $key = $1;
2148             my $value = $2;
2149             $conf->{$key} = $value;
2150         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2151             my $value = $1;
2152             if ($section eq 'pending') {
2153                 $conf->{delete} = $value; # we parse this later
2154             } else {
2155                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2156             }
2157         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2158             my $key = $1;
2159             my $value = $2;
2160             eval { $value = check_type($key, $value); };
2161             if ($@) {
2162                 warn "vm $vmid - unable to parse value of '$key' - $@";
2163             } else {
2164                 $key = 'ide2' if $key eq 'cdrom';
2165                 my $fmt = $confdesc->{$key}->{format};
2166                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2167                     my $v = parse_drive($key, $value);
2168                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2169                         $v->{file} = $volid;
2170                         $value = print_drive($vmid, $v);
2171                     } else {
2172                         warn "vm $vmid - unable to parse value of '$key'\n";
2173                         next;
2174                     }
2175                 }
2176
2177                 $conf->{$key} = $value;
2178             }
2179         }
2180     }
2181
2182     if (defined($descr)) {
2183         $descr =~ s/\s+$//;
2184         $conf->{description} = $descr;
2185     }
2186     delete $res->{snapstate}; # just to be sure
2187
2188     return $res;
2189 }
2190
2191 sub write_vm_config {
2192     my ($filename, $conf) = @_;
2193
2194     delete $conf->{snapstate}; # just to be sure
2195
2196     if ($conf->{cdrom}) {
2197         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2198         $conf->{ide2} = $conf->{cdrom};
2199         delete $conf->{cdrom};
2200     }
2201
2202     # we do not use 'smp' any longer
2203     if ($conf->{sockets}) {
2204         delete $conf->{smp};
2205     } elsif ($conf->{smp}) {
2206         $conf->{sockets} = $conf->{smp};
2207         delete $conf->{cores};
2208         delete $conf->{smp};
2209     }
2210
2211     my $used_volids = {};
2212
2213     my $cleanup_config = sub {
2214         my ($cref, $pending, $snapname) = @_;
2215
2216         foreach my $key (keys %$cref) {
2217             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2218                 $key eq 'snapstate' || $key eq 'pending';
2219             my $value = $cref->{$key};
2220             if ($key eq 'delete') {
2221                 die "propertry 'delete' is only allowed in [PENDING]\n"
2222                     if !$pending;
2223                 # fixme: check syntax?
2224                 next;
2225             }
2226             eval { $value = check_type($key, $value); };
2227             die "unable to parse value of '$key' - $@" if $@;
2228
2229             $cref->{$key} = $value;
2230
2231             if (!$snapname && is_valid_drivename($key)) {
2232                 my $drive = parse_drive($key, $value);
2233                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2234             }
2235         }
2236     };
2237
2238     &$cleanup_config($conf);
2239
2240     &$cleanup_config($conf->{pending}, 1);
2241
2242     foreach my $snapname (keys %{$conf->{snapshots}}) {
2243         die "internal error" if $snapname eq 'pending';
2244         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2245     }
2246
2247     # remove 'unusedX' settings if we re-add a volume
2248     foreach my $key (keys %$conf) {
2249         my $value = $conf->{$key};
2250         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2251             delete $conf->{$key};
2252         }
2253     }
2254
2255     my $generate_raw_config = sub {
2256         my ($conf, $pending) = @_;
2257
2258         my $raw = '';
2259
2260         # add description as comment to top of file
2261         if (defined(my $descr = $conf->{description})) {
2262             if ($descr) {
2263                 foreach my $cl (split(/\n/, $descr)) {
2264                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2265                 }
2266             } else {
2267                 $raw .= "#\n" if $pending;
2268             }
2269         }
2270
2271         foreach my $key (sort keys %$conf) {
2272             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2273             $raw .= "$key: $conf->{$key}\n";
2274         }
2275         return $raw;
2276     };
2277
2278     my $raw = &$generate_raw_config($conf);
2279
2280     if (scalar(keys %{$conf->{pending}})){
2281         $raw .= "\n[PENDING]\n";
2282         $raw .= &$generate_raw_config($conf->{pending}, 1);
2283     }
2284
2285     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2286         $raw .= "\n[$snapname]\n";
2287         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2288     }
2289
2290     return $raw;
2291 }
2292
2293 sub load_defaults {
2294
2295     my $res = {};
2296
2297     # we use static defaults from our JSON schema configuration
2298     foreach my $key (keys %$confdesc) {
2299         if (defined(my $default = $confdesc->{$key}->{default})) {
2300             $res->{$key} = $default;
2301         }
2302     }
2303
2304     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2305     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2306
2307     return $res;
2308 }
2309
2310 sub config_list {
2311     my $vmlist = PVE::Cluster::get_vmlist();
2312     my $res = {};
2313     return $res if !$vmlist || !$vmlist->{ids};
2314     my $ids = $vmlist->{ids};
2315
2316     foreach my $vmid (keys %$ids) {
2317         my $d = $ids->{$vmid};
2318         next if !$d->{node} || $d->{node} ne $nodename;
2319         next if !$d->{type} || $d->{type} ne 'qemu';
2320         $res->{$vmid}->{exists} = 1;
2321     }
2322     return $res;
2323 }
2324
2325 # test if VM uses local resources (to prevent migration)
2326 sub check_local_resources {
2327     my ($conf, $noerr) = @_;
2328
2329     my $loc_res = 0;
2330
2331     $loc_res = 1 if $conf->{hostusb}; # old syntax
2332     $loc_res = 1 if $conf->{hostpci}; # old syntax
2333
2334     foreach my $k (keys %$conf) {
2335         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2336         # sockets are safe: they will recreated be on the target side post-migrate
2337         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2338         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2339     }
2340
2341     die "VM uses local resources\n" if $loc_res && !$noerr;
2342
2343     return $loc_res;
2344 }
2345
2346 # check if used storages are available on all nodes (use by migrate)
2347 sub check_storage_availability {
2348     my ($storecfg, $conf, $node) = @_;
2349
2350     foreach_drive($conf, sub {
2351         my ($ds, $drive) = @_;
2352
2353         my $volid = $drive->{file};
2354         return if !$volid;
2355
2356         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2357         return if !$sid;
2358
2359         # check if storage is available on both nodes
2360         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2361         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2362    });
2363 }
2364
2365 # list nodes where all VM images are available (used by has_feature API)
2366 sub shared_nodes {
2367     my ($conf, $storecfg) = @_;
2368
2369     my $nodelist = PVE::Cluster::get_nodelist();
2370     my $nodehash = { map { $_ => 1 } @$nodelist };
2371     my $nodename = PVE::INotify::nodename();
2372
2373     foreach_drive($conf, sub {
2374         my ($ds, $drive) = @_;
2375
2376         my $volid = $drive->{file};
2377         return if !$volid;
2378
2379         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2380         if ($storeid) {
2381             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2382             if ($scfg->{disable}) {
2383                 $nodehash = {};
2384             } elsif (my $avail = $scfg->{nodes}) {
2385                 foreach my $node (keys %$nodehash) {
2386                     delete $nodehash->{$node} if !$avail->{$node};
2387                 }
2388             } elsif (!$scfg->{shared}) {
2389                 foreach my $node (keys %$nodehash) {
2390                     delete $nodehash->{$node} if $node ne $nodename
2391                 }
2392             }
2393         }
2394     });
2395
2396     return $nodehash
2397 }
2398
2399 sub check_cmdline {
2400     my ($pidfile, $pid) = @_;
2401
2402     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2403     if (defined($fh)) {
2404         my $line = <$fh>;
2405         $fh->close;
2406         return undef if !$line;
2407         my @param = split(/\0/, $line);
2408
2409         my $cmd = $param[0];
2410         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2411
2412         for (my $i = 0; $i < scalar (@param); $i++) {
2413             my $p = $param[$i];
2414             next if !$p;
2415             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2416                 my $p = $param[$i+1];
2417                 return 1 if $p && ($p eq $pidfile);
2418                 return undef;
2419             }
2420         }
2421     }
2422     return undef;
2423 }
2424
2425 sub check_running {
2426     my ($vmid, $nocheck, $node) = @_;
2427
2428     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2429
2430     die "unable to find configuration file for VM $vmid - no such machine\n"
2431         if !$nocheck && ! -f $filename;
2432
2433     my $pidfile = pidfile_name($vmid);
2434
2435     if (my $fd = IO::File->new("<$pidfile")) {
2436         my $st = stat($fd);
2437         my $line = <$fd>;
2438         close($fd);
2439
2440         my $mtime = $st->mtime;
2441         if ($mtime > time()) {
2442             warn "file '$filename' modified in future\n";
2443         }
2444
2445         if ($line =~ m/^(\d+)$/) {
2446             my $pid = $1;
2447             if (check_cmdline($pidfile, $pid)) {
2448                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2449                     return $pid;
2450                 }
2451             }
2452         }
2453     }
2454
2455     return undef;
2456 }
2457
2458 sub vzlist {
2459
2460     my $vzlist = config_list();
2461
2462     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2463
2464     while (defined(my $de = $fd->read)) {
2465         next if $de !~ m/^(\d+)\.pid$/;
2466         my $vmid = $1;
2467         next if !defined($vzlist->{$vmid});
2468         if (my $pid = check_running($vmid)) {
2469             $vzlist->{$vmid}->{pid} = $pid;
2470         }
2471     }
2472
2473     return $vzlist;
2474 }
2475
2476 sub disksize {
2477     my ($storecfg, $conf) = @_;
2478
2479     my $bootdisk = $conf->{bootdisk};
2480     return undef if !$bootdisk;
2481     return undef if !is_valid_drivename($bootdisk);
2482
2483     return undef if !$conf->{$bootdisk};
2484
2485     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2486     return undef if !defined($drive);
2487
2488     return undef if drive_is_cdrom($drive);
2489
2490     my $volid = $drive->{file};
2491     return undef if !$volid;
2492
2493     return $drive->{size};
2494 }
2495
2496 my $last_proc_pid_stat;
2497
2498 # get VM status information
2499 # This must be fast and should not block ($full == false)
2500 # We only query KVM using QMP if $full == true (this can be slow)
2501 sub vmstatus {
2502     my ($opt_vmid, $full) = @_;
2503
2504     my $res = {};
2505
2506     my $storecfg = PVE::Storage::config();
2507
2508     my $list = vzlist();
2509     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2510
2511     my $cpucount = $cpuinfo->{cpus} || 1;
2512
2513     foreach my $vmid (keys %$list) {
2514         next if $opt_vmid && ($vmid ne $opt_vmid);
2515
2516         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2517         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2518
2519         my $d = {};
2520         $d->{pid} = $list->{$vmid}->{pid};
2521
2522         # fixme: better status?
2523         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2524
2525         my $size = disksize($storecfg, $conf);
2526         if (defined($size)) {
2527             $d->{disk} = 0; # no info available
2528             $d->{maxdisk} = $size;
2529         } else {
2530             $d->{disk} = 0;
2531             $d->{maxdisk} = 0;
2532         }
2533
2534         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2535         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2536         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2537
2538         $d->{name} = $conf->{name} || "VM $vmid";
2539         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2540
2541         if ($conf->{balloon}) {
2542             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2543             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2544         }
2545
2546         $d->{uptime} = 0;
2547         $d->{cpu} = 0;
2548         $d->{mem} = 0;
2549
2550         $d->{netout} = 0;
2551         $d->{netin} = 0;
2552
2553         $d->{diskread} = 0;
2554         $d->{diskwrite} = 0;
2555
2556         $d->{template} = PVE::QemuConfig->is_template($conf);
2557
2558         $res->{$vmid} = $d;
2559     }
2560
2561     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2562     foreach my $dev (keys %$netdev) {
2563         next if $dev !~ m/^tap([1-9]\d*)i/;
2564         my $vmid = $1;
2565         my $d = $res->{$vmid};
2566         next if !$d;
2567
2568         $d->{netout} += $netdev->{$dev}->{receive};
2569         $d->{netin} += $netdev->{$dev}->{transmit};
2570
2571         if ($full) {
2572             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2573             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2574         }
2575
2576     }
2577
2578     my $ctime = gettimeofday;
2579
2580     foreach my $vmid (keys %$list) {
2581
2582         my $d = $res->{$vmid};
2583         my $pid = $d->{pid};
2584         next if !$pid;
2585
2586         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2587         next if !$pstat; # not running
2588
2589         my $used = $pstat->{utime} + $pstat->{stime};
2590
2591         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2592
2593         if ($pstat->{vsize}) {
2594             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2595         }
2596
2597         my $old = $last_proc_pid_stat->{$pid};
2598         if (!$old) {
2599             $last_proc_pid_stat->{$pid} = {
2600                 time => $ctime,
2601                 used => $used,
2602                 cpu => 0,
2603             };
2604             next;
2605         }
2606
2607         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2608
2609         if ($dtime > 1000) {
2610             my $dutime = $used -  $old->{used};
2611
2612             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2613             $last_proc_pid_stat->{$pid} = {
2614                 time => $ctime,
2615                 used => $used,
2616                 cpu => $d->{cpu},
2617             };
2618         } else {
2619             $d->{cpu} = $old->{cpu};
2620         }
2621     }
2622
2623     return $res if !$full;
2624
2625     my $qmpclient = PVE::QMPClient->new();
2626
2627     my $ballooncb = sub {
2628         my ($vmid, $resp) = @_;
2629
2630         my $info = $resp->{'return'};
2631         return if !$info->{max_mem};
2632
2633         my $d = $res->{$vmid};
2634
2635         # use memory assigned to VM
2636         $d->{maxmem} = $info->{max_mem};
2637         $d->{balloon} = $info->{actual};
2638
2639         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2640             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2641             $d->{freemem} = $info->{free_mem};
2642         }
2643
2644         $d->{ballooninfo} = $info;
2645     };
2646
2647     my $blockstatscb = sub {
2648         my ($vmid, $resp) = @_;
2649         my $data = $resp->{'return'} || [];
2650         my $totalrdbytes = 0;
2651         my $totalwrbytes = 0;
2652
2653         for my $blockstat (@$data) {
2654             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2655             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2656
2657             $blockstat->{device} =~ s/drive-//;
2658             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2659         }
2660         $res->{$vmid}->{diskread} = $totalrdbytes;
2661         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2662     };
2663
2664     my $statuscb = sub {
2665         my ($vmid, $resp) = @_;
2666
2667         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2668         # this fails if ballon driver is not loaded, so this must be
2669         # the last commnand (following command are aborted if this fails).
2670         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2671
2672         my $status = 'unknown';
2673         if (!defined($status = $resp->{'return'}->{status})) {
2674             warn "unable to get VM status\n";
2675             return;
2676         }
2677
2678         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2679     };
2680
2681     foreach my $vmid (keys %$list) {
2682         next if $opt_vmid && ($vmid ne $opt_vmid);
2683         next if !$res->{$vmid}->{pid}; # not running
2684         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2685     }
2686
2687     $qmpclient->queue_execute(undef, 1);
2688
2689     foreach my $vmid (keys %$list) {
2690         next if $opt_vmid && ($vmid ne $opt_vmid);
2691         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2692     }
2693
2694     return $res;
2695 }
2696
2697 sub foreach_drive {
2698     my ($conf, $func, @param) = @_;
2699
2700     foreach my $ds (valid_drive_names()) {
2701         next if !defined($conf->{$ds});
2702
2703         my $drive = parse_drive($ds, $conf->{$ds});
2704         next if !$drive;
2705
2706         &$func($ds, $drive, @param);
2707     }
2708 }
2709
2710 sub foreach_volid {
2711     my ($conf, $func, @param) = @_;
2712
2713     my $volhash = {};
2714
2715     my $test_volid = sub {
2716         my ($volid, $is_cdrom) = @_;
2717
2718         return if !$volid;
2719
2720         $volhash->{$volid} = $is_cdrom || 0;
2721     };
2722
2723     foreach_drive($conf, sub {
2724         my ($ds, $drive) = @_;
2725         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2726     });
2727
2728     foreach my $snapname (keys %{$conf->{snapshots}}) {
2729         my $snap = $conf->{snapshots}->{$snapname};
2730         &$test_volid($snap->{vmstate}, 0);
2731         foreach_drive($snap, sub {
2732             my ($ds, $drive) = @_;
2733             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2734         });
2735     }
2736
2737     foreach my $volid (keys %$volhash) {
2738         &$func($volid, $volhash->{$volid}, @param);
2739     }
2740 }
2741
2742 sub vga_conf_has_spice {
2743     my ($vga) = @_;
2744
2745     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2746
2747     return $1 || 1;
2748 }
2749
2750 sub config_to_command {
2751     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2752
2753     my $cmd = [];
2754     my $globalFlags = [];
2755     my $machineFlags = [];
2756     my $rtcFlags = [];
2757     my $cpuFlags = [];
2758     my $devices = [];
2759     my $pciaddr = '';
2760     my $bridges = {};
2761     my $kvmver = kvm_user_version();
2762     my $vernum = 0; # unknown
2763     my $ostype = $conf->{ostype};
2764     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2765         $vernum = $1*1000000+$2*1000;
2766     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2767         $vernum = $1*1000000+$2*1000+$3;
2768     }
2769
2770     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2771
2772     my $have_ovz = -f '/proc/vz/vestat';
2773
2774     my $q35 = machine_type_is_q35($conf);
2775     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2776     my $machine_type = $forcemachine || $conf->{machine};
2777     my $use_old_bios_files = undef;
2778     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2779
2780     my $cpuunits = defined($conf->{cpuunits}) ?
2781             $conf->{cpuunits} : $defaults->{cpuunits};
2782
2783     push @$cmd, '/usr/bin/kvm';
2784
2785     push @$cmd, '-id', $vmid;
2786
2787     my $use_virtio = 0;
2788
2789     my $qmpsocket = qmp_socket($vmid);
2790     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2791     push @$cmd, '-mon', "chardev=qmp,mode=control";
2792
2793
2794     push @$cmd, '-pidfile' , pidfile_name($vmid);
2795
2796     push @$cmd, '-daemonize';
2797
2798     if ($conf->{smbios1}) {
2799         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2800     }
2801
2802     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2803         my $ovmfbase;
2804
2805         # prefer the OVMF_CODE variant
2806         if (-f $OVMF_CODE) {
2807             $ovmfbase = $OVMF_CODE;
2808         } elsif (-f $OVMF_IMG) {
2809             $ovmfbase = $OVMF_IMG;
2810         }
2811
2812         die "no uefi base img found\n" if !$ovmfbase;
2813         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2814
2815         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2816             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2817             my $format = $d->{format} // 'raw';
2818             my $path;
2819             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2820             if ($storeid) {
2821                 $path = PVE::Storage::path($storecfg, $d->{file});
2822                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2823                 $format = qemu_img_format($scfg, $volname);
2824             } else {
2825                 $path = $d->{file};
2826                 $format = "raw";
2827             }
2828             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2829         } elsif ($ovmfbase eq $OVMF_CODE) {
2830             warn "using uefi without permanent efivars disk\n";
2831             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2832             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2833             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2834         } else {
2835             # if the base img is not OVMF_CODE, we do not have to bother
2836             # to create/use a vars image, since it will not be used anyway
2837             # this can only happen if someone manually deletes the OVMF_CODE image
2838             # or has an old pve-qemu-kvm version installed.
2839             # both should not happen, but we ignore it here
2840         }
2841     }
2842
2843
2844     # add usb controllers
2845     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2846     push @$devices, @usbcontrollers if @usbcontrollers;
2847     my $vga = $conf->{vga};
2848
2849     my $qxlnum = vga_conf_has_spice($vga);
2850     $vga = 'qxl' if $qxlnum;
2851
2852     if (!$vga) {
2853         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2854                                 $conf->{ostype} eq 'win7' ||
2855                                 $conf->{ostype} eq 'w2k8')) {
2856             $vga = 'std';
2857         } else {
2858             $vga = 'cirrus';
2859         }
2860     }
2861
2862     # enable absolute mouse coordinates (needed by vnc)
2863     my $tablet;
2864     if (defined($conf->{tablet})) {
2865         $tablet = $conf->{tablet};
2866     } else {
2867         $tablet = $defaults->{tablet};
2868         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2869         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2870     }
2871
2872     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2873
2874     my $kvm_off = 0;
2875     # host pci devices
2876     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2877         my $d = parse_hostpci($conf->{"hostpci$i"});
2878         next if !$d;
2879
2880         my $pcie = $d->{pcie};
2881         if($pcie){
2882             die "q35 machine model is not enabled" if !$q35;
2883             $pciaddr = print_pcie_addr("hostpci$i");
2884         }else{
2885             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2886         }
2887
2888         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2889         my $xvga = '';
2890         if ($d->{'x-vga'}) {
2891             $xvga = ',x-vga=on';
2892             $kvm_off = 1;
2893             $vga = 'none';
2894             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2895                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2896             }
2897             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2898                 $xvga = "";
2899             }
2900         }
2901         my $pcidevices = $d->{pciid};
2902         my $multifunction = 1 if @$pcidevices > 1;
2903
2904         my $j=0;
2905         foreach my $pcidevice (@$pcidevices) {
2906
2907             my $id = "hostpci$i";
2908             $id .= ".$j" if $multifunction;
2909             my $addr = $pciaddr;
2910             $addr .= ".$j" if $multifunction;
2911             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2912
2913             if($j == 0){
2914                 $devicestr .= "$rombar$xvga";
2915                 $devicestr .= ",multifunction=on" if $multifunction;
2916             }
2917
2918             push @$devices, '-device', $devicestr;
2919             $j++;
2920         }
2921     }
2922
2923     # usb devices
2924     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2925     push @$devices, @usbdevices if @usbdevices;
2926     # serial devices
2927     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2928         if (my $path = $conf->{"serial$i"}) {
2929             if ($path eq 'socket') {
2930                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2931                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2932                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2933             } else {
2934                 die "no such serial device\n" if ! -c $path;
2935                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2936                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2937             }
2938         }
2939     }
2940
2941     # parallel devices
2942     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2943         if (my $path = $conf->{"parallel$i"}) {
2944             die "no such parallel device\n" if ! -c $path;
2945             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2946             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2947             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2948         }
2949     }
2950
2951     my $vmname = $conf->{name} || "vm$vmid";
2952
2953     push @$cmd, '-name', $vmname;
2954
2955     my $sockets = 1;
2956     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2957     $sockets = $conf->{sockets} if  $conf->{sockets};
2958
2959     my $cores = $conf->{cores} || 1;
2960
2961     my $maxcpus = $sockets * $cores;
2962
2963     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2964
2965     my $allowed_vcpus = $cpuinfo->{cpus};
2966
2967     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2968         if ($allowed_vcpus < $maxcpus);
2969
2970     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
2971
2972         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2973         for (my $i = 2; $i <= $vcpus; $i++)  {
2974             my $cpustr = print_cpu_device($conf,$i);
2975             push @$cmd, '-device', $cpustr;
2976         }
2977
2978     } else {
2979
2980         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2981     }
2982     push @$cmd, '-nodefaults';
2983
2984     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2985
2986     my $bootindex_hash = {};
2987     my $i = 1;
2988     foreach my $o (split(//, $bootorder)) {
2989         $bootindex_hash->{$o} = $i*100;
2990         $i++;
2991     }
2992
2993     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
2994
2995     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2996
2997     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2998
2999     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3000
3001     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3002         my $socket = vnc_socket($vmid);
3003         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3004     } else {
3005         push @$cmd, '-nographic';
3006     }
3007
3008     # time drift fix
3009     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3010
3011     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3012     my $useLocaltime = $conf->{localtime};
3013
3014     if ($ostype) {
3015         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3016
3017         if ($ostype =~ m/^w/) { # windows
3018             $useLocaltime = 1 if !defined($conf->{localtime});
3019
3020             # use time drift fix when acpi is enabled
3021             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3022                 $tdf = 1 if !defined($conf->{tdf});
3023             }
3024         }
3025
3026         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3027             $ostype eq 'wvista') {
3028             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3029             push @$cmd, '-no-hpet';
3030             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3031                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3032                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3033                 push @$cpuFlags , 'hv_time' if !$nokvm;
3034
3035                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
3036                     push @$cpuFlags , 'hv_reset' if !$nokvm;
3037                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
3038                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
3039                 }
3040
3041             } else {
3042                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3043             }
3044         }
3045
3046         if ($ostype eq 'win7' || $ostype eq 'win8') {
3047             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3048         }
3049     }
3050
3051     push @$rtcFlags, 'driftfix=slew' if $tdf;
3052
3053     if ($nokvm) {
3054         push @$machineFlags, 'accel=tcg';
3055     } else {
3056         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3057     }
3058
3059     if ($machine_type) {
3060         push @$machineFlags, "type=${machine_type}";
3061     }
3062
3063     if ($conf->{startdate}) {
3064         push @$rtcFlags, "base=$conf->{startdate}";
3065     } elsif ($useLocaltime) {
3066         push @$rtcFlags, 'base=localtime';
3067     }
3068
3069     my $cpu = $nokvm ? "qemu64" : "kvm64";
3070     if (my $cputype = $conf->{cpu}) {
3071         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3072             or die "Cannot parse cpu description: $cputype\n";
3073         $cpu = $cpuconf->{cputype};
3074         $kvm_off = 1 if $cpuconf->{hidden};
3075     }
3076
3077     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3078
3079     push @$cpuFlags , '-x2apic'
3080         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3081
3082     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3083
3084     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3085
3086     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3087
3088         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3089         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3090     }
3091
3092     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3093
3094     push @$cpuFlags, 'kvm=off' if $kvm_off;
3095
3096     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3097         die "internal error"; # should not happen
3098
3099     push @$cpuFlags, "vendor=${cpu_vendor}"
3100         if $cpu_vendor ne 'default';
3101
3102     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3103
3104     push @$cmd, '-cpu', $cpu;
3105
3106     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3107
3108     push @$cmd, '-S' if $conf->{freeze};
3109
3110     # set keyboard layout
3111     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3112     push @$cmd, '-k', $kb if $kb;
3113
3114     # enable sound
3115     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3116     #push @$cmd, '-soundhw', 'es1370';
3117     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3118
3119     if($conf->{agent}) {
3120         my $qgasocket = qmp_socket($vmid, 1);
3121         my $pciaddr = print_pci_addr("qga0", $bridges);
3122         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3123         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3124         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3125     }
3126
3127     my $spice_port;
3128
3129     if ($qxlnum) {
3130         if ($qxlnum > 1) {
3131             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3132                 for(my $i = 1; $i < $qxlnum; $i++){
3133                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3134                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3135                 }
3136             } else {
3137                 # assume other OS works like Linux
3138                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3139                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3140             }
3141         }
3142
3143         my $pciaddr = print_pci_addr("spice", $bridges);
3144
3145         my $nodename = PVE::INotify::nodename();
3146         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3147         $spice_port = PVE::Tools::next_spice_port($pfamily);
3148
3149         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3150
3151         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3152         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3153         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3154     }
3155
3156     # enable balloon by default, unless explicitly disabled
3157     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3158         $pciaddr = print_pci_addr("balloon0", $bridges);
3159         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3160     }
3161
3162     if ($conf->{watchdog}) {
3163         my $wdopts = parse_watchdog($conf->{watchdog});
3164         $pciaddr = print_pci_addr("watchdog", $bridges);
3165         my $watchdog = $wdopts->{model} || 'i6300esb';
3166         push @$devices, '-device', "$watchdog$pciaddr";
3167         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3168     }
3169
3170     my $vollist = [];
3171     my $scsicontroller = {};
3172     my $ahcicontroller = {};
3173     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3174
3175     # Add iscsi initiator name if available
3176     if (my $initiator = get_initiator_name()) {
3177         push @$devices, '-iscsi', "initiator-name=$initiator";
3178     }
3179
3180     foreach_drive($conf, sub {
3181         my ($ds, $drive) = @_;
3182
3183         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3184             push @$vollist, $drive->{file};
3185         }
3186
3187         $use_virtio = 1 if $ds =~ m/^virtio/;
3188
3189         if (drive_is_cdrom ($drive)) {
3190             if ($bootindex_hash->{d}) {
3191                 $drive->{bootindex} = $bootindex_hash->{d};
3192                 $bootindex_hash->{d} += 1;
3193             }
3194         } else {
3195             if ($bootindex_hash->{c}) {
3196                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3197                 $bootindex_hash->{c} += 1;
3198             }
3199         }
3200
3201         if($drive->{interface} eq 'virtio'){
3202            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3203         }
3204
3205         if ($drive->{interface} eq 'scsi') {
3206
3207             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3208
3209             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3210             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3211
3212             my $iothread = '';
3213             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3214                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3215                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3216             } elsif ($drive->{iothread}) {
3217                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3218             }
3219
3220             my $queues = '';
3221             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3222                 $queues = ",num_queues=$drive->{queues}";
3223             }
3224
3225             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3226             $scsicontroller->{$controller}=1;
3227         }
3228
3229         if ($drive->{interface} eq 'sata') {
3230            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3231            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3232            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3233            $ahcicontroller->{$controller}=1;
3234         }
3235
3236         if ($drive->{interface} eq 'efidisk') {
3237             # this will be added somewhere else
3238             return;
3239         }
3240
3241         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3242         push @$devices, '-drive',$drive_cmd;
3243         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3244     });
3245
3246     for (my $i = 0; $i < $MAX_NETS; $i++) {
3247          next if !$conf->{"net$i"};
3248          my $d = parse_net($conf->{"net$i"});
3249          next if !$d;
3250
3251          $use_virtio = 1 if $d->{model} eq 'virtio';
3252
3253          if ($bootindex_hash->{n}) {
3254             $d->{bootindex} = $bootindex_hash->{n};
3255             $bootindex_hash->{n} += 1;
3256          }
3257
3258          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3259          push @$devices, '-netdev', $netdevfull;
3260
3261          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3262          push @$devices, '-device', $netdevicefull;
3263     }
3264
3265     if (!$q35) {
3266         # add pci bridges
3267         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3268            $bridges->{1} = 1;
3269            $bridges->{2} = 1;
3270         }
3271
3272         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3273
3274         while (my ($k, $v) = each %$bridges) {
3275             $pciaddr = print_pci_addr("pci.$k");
3276             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3277         }
3278     }
3279
3280     # add custom args
3281     if ($conf->{args}) {
3282         my $aa = PVE::Tools::split_args($conf->{args});
3283         push @$cmd, @$aa;
3284     }
3285
3286     push @$cmd, @$devices;
3287     push @$cmd, '-rtc', join(',', @$rtcFlags)
3288         if scalar(@$rtcFlags);
3289     push @$cmd, '-machine', join(',', @$machineFlags)
3290         if scalar(@$machineFlags);
3291     push @$cmd, '-global', join(',', @$globalFlags)
3292         if scalar(@$globalFlags);
3293
3294     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3295 }
3296
3297 sub vnc_socket {
3298     my ($vmid) = @_;
3299     return "${var_run_tmpdir}/$vmid.vnc";
3300 }
3301
3302 sub spice_port {
3303     my ($vmid) = @_;
3304
3305     my $res = vm_mon_cmd($vmid, 'query-spice');
3306
3307     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3308 }
3309
3310 sub qmp_socket {
3311     my ($vmid, $qga) = @_;
3312     my $sockettype = $qga ? 'qga' : 'qmp';
3313     return "${var_run_tmpdir}/$vmid.$sockettype";
3314 }
3315
3316 sub pidfile_name {
3317     my ($vmid) = @_;
3318     return "${var_run_tmpdir}/$vmid.pid";
3319 }
3320
3321 sub vm_devices_list {
3322     my ($vmid) = @_;
3323
3324     my $res = vm_mon_cmd($vmid, 'query-pci');
3325     my $devices = {};
3326     foreach my $pcibus (@$res) {
3327         foreach my $device (@{$pcibus->{devices}}) {
3328             next if !$device->{'qdev_id'};
3329             if ($device->{'pci_bridge'}) {
3330                 $devices->{$device->{'qdev_id'}} = 1;
3331                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3332                     next if !$bridge_device->{'qdev_id'};
3333                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3334                     $devices->{$device->{'qdev_id'}}++;
3335                 }
3336             } else {
3337                 $devices->{$device->{'qdev_id'}} = 1;
3338             }
3339         }
3340     }
3341
3342     my $resblock = vm_mon_cmd($vmid, 'query-block');
3343     foreach my $block (@$resblock) {
3344         if($block->{device} =~ m/^drive-(\S+)/){
3345                 $devices->{$1} = 1;
3346         }
3347     }
3348
3349     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3350     foreach my $mice (@$resmice) {
3351         if ($mice->{name} eq 'QEMU HID Tablet') {
3352             $devices->{tablet} = 1;
3353             last;
3354         }
3355     }
3356
3357     # for usb devices there is no query-usb
3358     # but we can iterate over the entries in
3359     # qom-list path=/machine/peripheral
3360     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3361     foreach my $per (@$resperipheral) {
3362         if ($per->{name} =~ m/^usb\d+$/) {
3363             $devices->{$per->{name}} = 1;
3364         }
3365     }
3366
3367     return $devices;
3368 }
3369
3370 sub vm_deviceplug {
3371     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3372
3373     my $q35 = machine_type_is_q35($conf);
3374
3375     my $devices_list = vm_devices_list($vmid);
3376     return 1 if defined($devices_list->{$deviceid});
3377
3378     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3379
3380     if ($deviceid eq 'tablet') {
3381
3382         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3383
3384     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3385
3386         die "usb hotplug currently not reliable\n";
3387         # since we can't reliably hot unplug all added usb devices
3388         # and usb passthrough disables live migration
3389         # we disable usb hotplugging for now
3390         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3391
3392     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3393
3394         qemu_iothread_add($vmid, $deviceid, $device);
3395
3396         qemu_driveadd($storecfg, $vmid, $device);
3397         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3398
3399         qemu_deviceadd($vmid, $devicefull);
3400         eval { qemu_deviceaddverify($vmid, $deviceid); };
3401         if (my $err = $@) {
3402             eval { qemu_drivedel($vmid, $deviceid); };
3403             warn $@ if $@;
3404             die $err;
3405         }
3406
3407     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3408
3409
3410         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3411         my $pciaddr = print_pci_addr($deviceid);
3412         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3413
3414         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3415
3416         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3417             qemu_iothread_add($vmid, $deviceid, $device);
3418             $devicefull .= ",iothread=iothread-$deviceid";
3419         }
3420
3421         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3422             $devicefull .= ",num_queues=$device->{queues}";
3423         }
3424
3425         qemu_deviceadd($vmid, $devicefull);
3426         qemu_deviceaddverify($vmid, $deviceid);
3427
3428     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3429
3430         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3431         qemu_driveadd($storecfg, $vmid, $device);
3432
3433         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3434         eval { qemu_deviceadd($vmid, $devicefull); };
3435         if (my $err = $@) {
3436             eval { qemu_drivedel($vmid, $deviceid); };
3437             warn $@ if $@;
3438             die $err;
3439         }
3440
3441     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3442
3443         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3444
3445         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3446         my $use_old_bios_files = undef;
3447         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3448
3449         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3450         qemu_deviceadd($vmid, $netdevicefull);
3451         eval { qemu_deviceaddverify($vmid, $deviceid); };
3452         if (my $err = $@) {
3453             eval { qemu_netdevdel($vmid, $deviceid); };
3454             warn $@ if $@;
3455             die $err;
3456         }
3457
3458     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3459
3460         my $bridgeid = $2;
3461         my $pciaddr = print_pci_addr($deviceid);
3462         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3463
3464         qemu_deviceadd($vmid, $devicefull);
3465         qemu_deviceaddverify($vmid, $deviceid);
3466
3467     } else {
3468         die "can't hotplug device '$deviceid'\n";
3469     }
3470
3471     return 1;
3472 }
3473
3474 # fixme: this should raise exceptions on error!
3475 sub vm_deviceunplug {
3476     my ($vmid, $conf, $deviceid) = @_;
3477
3478     my $devices_list = vm_devices_list($vmid);
3479     return 1 if !defined($devices_list->{$deviceid});
3480
3481     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3482
3483     if ($deviceid eq 'tablet') {
3484
3485         qemu_devicedel($vmid, $deviceid);
3486
3487     } elsif ($deviceid =~ m/^usb\d+$/) {
3488
3489         die "usb hotplug currently not reliable\n";
3490         # when unplugging usb devices this way,
3491         # there may be remaining usb controllers/hubs
3492         # so we disable it for now
3493         qemu_devicedel($vmid, $deviceid);
3494         qemu_devicedelverify($vmid, $deviceid);
3495
3496     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3497
3498         qemu_devicedel($vmid, $deviceid);
3499         qemu_devicedelverify($vmid, $deviceid);
3500         qemu_drivedel($vmid, $deviceid);
3501         qemu_iothread_del($conf, $vmid, $deviceid);
3502
3503     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3504
3505         qemu_devicedel($vmid, $deviceid);
3506         qemu_devicedelverify($vmid, $deviceid);
3507         qemu_iothread_del($conf, $vmid, $deviceid);
3508
3509     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3510
3511         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3512         my $device = parse_drive($deviceid, $conf->{$deviceid});
3513         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3514
3515         qemu_devicedel($vmid, $deviceid);
3516         qemu_drivedel($vmid, $deviceid);
3517         qemu_deletescsihw($conf, $vmid, $deviceid);
3518
3519     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3520
3521         qemu_devicedel($vmid, $deviceid);
3522         qemu_devicedelverify($vmid, $deviceid);
3523         qemu_netdevdel($vmid, $deviceid);
3524
3525     } else {
3526         die "can't unplug device '$deviceid'\n";
3527     }
3528
3529     return 1;
3530 }
3531
3532 sub qemu_deviceadd {
3533     my ($vmid, $devicefull) = @_;
3534
3535     $devicefull = "driver=".$devicefull;
3536     my %options =  split(/[=,]/, $devicefull);
3537
3538     vm_mon_cmd($vmid, "device_add" , %options);
3539 }
3540
3541 sub qemu_devicedel {
3542     my ($vmid, $deviceid) = @_;
3543
3544     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3545 }
3546
3547 sub qemu_iothread_add {
3548     my($vmid, $deviceid, $device) = @_;
3549
3550     if ($device->{iothread}) {
3551         my $iothreads = vm_iothreads_list($vmid);
3552         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3553     }
3554 }
3555
3556 sub qemu_iothread_del {
3557     my($conf, $vmid, $deviceid) = @_;
3558
3559     my $device = parse_drive($deviceid, $conf->{$deviceid});
3560     if ($device->{iothread}) {
3561         my $iothreads = vm_iothreads_list($vmid);
3562         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3563     }
3564 }
3565
3566 sub qemu_objectadd {
3567     my($vmid, $objectid, $qomtype) = @_;
3568
3569     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3570
3571     return 1;
3572 }
3573
3574 sub qemu_objectdel {
3575     my($vmid, $objectid) = @_;
3576
3577     vm_mon_cmd($vmid, "object-del", id => $objectid);
3578
3579     return 1;
3580 }
3581
3582 sub qemu_driveadd {
3583     my ($storecfg, $vmid, $device) = @_;
3584
3585     my $drive = print_drive_full($storecfg, $vmid, $device);
3586     $drive =~ s/\\/\\\\/g;
3587     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3588
3589     # If the command succeeds qemu prints: "OK"
3590     return 1 if $ret =~ m/OK/s;
3591
3592     die "adding drive failed: $ret\n";
3593 }
3594
3595 sub qemu_drivedel {
3596     my($vmid, $deviceid) = @_;
3597
3598     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3599     $ret =~ s/^\s+//;
3600
3601     return 1 if $ret eq "";
3602
3603     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3604     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3605
3606     die "deleting drive $deviceid failed : $ret\n";
3607 }
3608
3609 sub qemu_deviceaddverify {
3610     my ($vmid, $deviceid) = @_;
3611
3612     for (my $i = 0; $i <= 5; $i++) {
3613          my $devices_list = vm_devices_list($vmid);
3614          return 1 if defined($devices_list->{$deviceid});
3615          sleep 1;
3616     }
3617
3618     die "error on hotplug device '$deviceid'\n";
3619 }
3620
3621
3622 sub qemu_devicedelverify {
3623     my ($vmid, $deviceid) = @_;
3624
3625     # need to verify that the device is correctly removed as device_del
3626     # is async and empty return is not reliable
3627
3628     for (my $i = 0; $i <= 5; $i++) {
3629          my $devices_list = vm_devices_list($vmid);
3630          return 1 if !defined($devices_list->{$deviceid});
3631          sleep 1;
3632     }
3633
3634     die "error on hot-unplugging device '$deviceid'\n";
3635 }
3636
3637 sub qemu_findorcreatescsihw {
3638     my ($storecfg, $conf, $vmid, $device) = @_;
3639
3640     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3641
3642     my $scsihwid="$controller_prefix$controller";
3643     my $devices_list = vm_devices_list($vmid);
3644
3645     if(!defined($devices_list->{$scsihwid})) {
3646         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3647     }
3648
3649     return 1;
3650 }
3651
3652 sub qemu_deletescsihw {
3653     my ($conf, $vmid, $opt) = @_;
3654
3655     my $device = parse_drive($opt, $conf->{$opt});
3656
3657     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3658         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3659         return 1;
3660     }
3661
3662     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3663
3664     my $devices_list = vm_devices_list($vmid);
3665     foreach my $opt (keys %{$devices_list}) {
3666         if (PVE::QemuServer::is_valid_drivename($opt)) {
3667             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3668             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3669                 return 1;
3670             }
3671         }
3672     }
3673
3674     my $scsihwid="scsihw$controller";
3675
3676     vm_deviceunplug($vmid, $conf, $scsihwid);
3677
3678     return 1;
3679 }
3680
3681 sub qemu_add_pci_bridge {
3682     my ($storecfg, $conf, $vmid, $device) = @_;
3683
3684     my $bridges = {};
3685
3686     my $bridgeid;
3687
3688     print_pci_addr($device, $bridges);
3689
3690     while (my ($k, $v) = each %$bridges) {
3691         $bridgeid = $k;
3692     }
3693     return 1 if !defined($bridgeid) || $bridgeid < 1;
3694
3695     my $bridge = "pci.$bridgeid";
3696     my $devices_list = vm_devices_list($vmid);
3697
3698     if (!defined($devices_list->{$bridge})) {
3699         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3700     }
3701
3702     return 1;
3703 }
3704
3705 sub qemu_set_link_status {
3706     my ($vmid, $device, $up) = @_;
3707
3708     vm_mon_cmd($vmid, "set_link", name => $device,
3709                up => $up ? JSON::true : JSON::false);
3710 }
3711
3712 sub qemu_netdevadd {
3713     my ($vmid, $conf, $device, $deviceid) = @_;
3714
3715     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3716     my %options =  split(/[=,]/, $netdev);
3717
3718     vm_mon_cmd($vmid, "netdev_add",  %options);
3719     return 1;
3720 }
3721
3722 sub qemu_netdevdel {
3723     my ($vmid, $deviceid) = @_;
3724
3725     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3726 }
3727
3728 sub qemu_usb_hotplug {
3729     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3730
3731     return if !$device;
3732
3733     # remove the old one first
3734     vm_deviceunplug($vmid, $conf, $deviceid);
3735
3736     # check if xhci controller is necessary and available
3737     if ($device->{usb3}) {
3738
3739         my $devicelist = vm_devices_list($vmid);
3740
3741         if (!$devicelist->{xhci}) {
3742             my $pciaddr = print_pci_addr("xhci");
3743             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3744         }
3745     }
3746     my $d = parse_usb_device($device->{host});
3747     $d->{usb3} = $device->{usb3};
3748
3749     # add the new one
3750     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3751 }
3752
3753 sub qemu_cpu_hotplug {
3754     my ($vmid, $conf, $vcpus) = @_;
3755
3756     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3757
3758     my $sockets = 1;
3759     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3760     $sockets = $conf->{sockets} if  $conf->{sockets};
3761     my $cores = $conf->{cores} || 1;
3762     my $maxcpus = $sockets * $cores;
3763
3764     $vcpus = $maxcpus if !$vcpus;
3765
3766     die "you can't add more vcpus than maxcpus\n"
3767         if $vcpus > $maxcpus;
3768
3769     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3770
3771     if ($vcpus < $currentvcpus) {
3772
3773         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3774
3775             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3776                 qemu_devicedel($vmid, "cpu$i");
3777                 my $retry = 0;
3778                 my $currentrunningvcpus = undef;
3779                 while (1) {
3780                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3781                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3782                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3783                     $retry++;
3784                     sleep 1;
3785                 }
3786                 #update conf after each succesfull cpu unplug
3787                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3788                 PVE::QemuConfig->write_config($vmid, $conf);
3789             }
3790         } else {
3791             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3792         }
3793
3794         return;
3795     }
3796
3797     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3798     die "vcpus in running vm does not match its configuration\n"
3799         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3800
3801     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3802
3803         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3804             my $cpustr = print_cpu_device($conf, $i);
3805             qemu_deviceadd($vmid, $cpustr);
3806
3807             my $retry = 0;
3808             my $currentrunningvcpus = undef;
3809             while (1) {
3810                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3811                 last if scalar(@{$currentrunningvcpus}) == $i;
3812                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3813                 sleep 1;
3814                 $retry++;
3815             }
3816             #update conf after each succesfull cpu hotplug
3817             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3818             PVE::QemuConfig->write_config($vmid, $conf);
3819         }
3820     } else {
3821
3822         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3823             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3824         }
3825     }
3826 }
3827
3828 sub qemu_block_set_io_throttle {
3829     my ($vmid, $deviceid,
3830         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3831         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3832
3833     return if !check_running($vmid) ;
3834
3835     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3836         bps => int($bps),
3837         bps_rd => int($bps_rd),
3838         bps_wr => int($bps_wr),
3839         iops => int($iops),
3840         iops_rd => int($iops_rd),
3841         iops_wr => int($iops_wr),
3842         bps_max => int($bps_max),
3843         bps_rd_max => int($bps_rd_max),
3844         bps_wr_max => int($bps_wr_max),
3845         iops_max => int($iops_max),
3846         iops_rd_max => int($iops_rd_max),
3847         iops_wr_max => int($iops_wr_max)
3848     );
3849
3850 }
3851
3852 # old code, only used to shutdown old VM after update
3853 sub __read_avail {
3854     my ($fh, $timeout) = @_;
3855
3856     my $sel = new IO::Select;
3857     $sel->add($fh);
3858
3859     my $res = '';
3860     my $buf;
3861
3862     my @ready;
3863     while (scalar (@ready = $sel->can_read($timeout))) {
3864         my $count;
3865         if ($count = $fh->sysread($buf, 8192)) {
3866             if ($buf =~ /^(.*)\(qemu\) $/s) {
3867                 $res .= $1;
3868                 last;
3869             } else {
3870                 $res .= $buf;
3871             }
3872         } else {
3873             if (!defined($count)) {
3874                 die "$!\n";
3875             }
3876             last;
3877         }
3878     }
3879
3880     die "monitor read timeout\n" if !scalar(@ready);
3881
3882     return $res;
3883 }
3884
3885 # old code, only used to shutdown old VM after update
3886 sub vm_monitor_command {
3887     my ($vmid, $cmdstr, $nocheck) = @_;
3888
3889     my $res;
3890
3891     eval {
3892         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3893
3894         my $sname = "${var_run_tmpdir}/$vmid.mon";
3895
3896         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3897             die "unable to connect to VM $vmid socket - $!\n";
3898
3899         my $timeout = 3;
3900
3901         # hack: migrate sometime blocks the monitor (when migrate_downtime
3902         # is set)
3903         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3904             $timeout = 60*60; # 1 hour
3905         }
3906
3907         # read banner;
3908         my $data = __read_avail($sock, $timeout);
3909
3910         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3911             die "got unexpected qemu monitor banner\n";
3912         }
3913
3914         my $sel = new IO::Select;
3915         $sel->add($sock);
3916
3917         if (!scalar(my @ready = $sel->can_write($timeout))) {
3918             die "monitor write error - timeout";
3919         }
3920
3921         my $fullcmd = "$cmdstr\r";
3922
3923         # syslog('info', "VM $vmid monitor command: $cmdstr");
3924
3925         my $b;
3926         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3927             die "monitor write error - $!";
3928         }
3929
3930         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3931
3932         $timeout = 20;
3933
3934         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3935             $timeout = 60*60; # 1 hour
3936         } elsif ($cmdstr =~ m/^(eject|change)/) {
3937             $timeout = 60; # note: cdrom mount command is slow
3938         }
3939         if ($res = __read_avail($sock, $timeout)) {
3940
3941             my @lines = split("\r?\n", $res);
3942
3943             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3944
3945             $res = join("\n", @lines);
3946             $res .= "\n";
3947         }
3948     };
3949
3950     my $err = $@;
3951
3952     if ($err) {
3953         syslog("err", "VM $vmid monitor command failed - $err");
3954         die $err;
3955     }
3956
3957     return $res;
3958 }
3959
3960 sub qemu_block_resize {
3961     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3962
3963     my $running = check_running($vmid);
3964
3965     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3966
3967     return if !$running;
3968
3969     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3970
3971 }
3972
3973 sub qemu_volume_snapshot {
3974     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3975
3976     my $running = check_running($vmid);
3977
3978     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3979         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3980     } else {
3981         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3982     }
3983 }
3984
3985 sub qemu_volume_snapshot_delete {
3986     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3987
3988     my $running = check_running($vmid);
3989
3990     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3991
3992     return if !$running;
3993
3994     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3995 }
3996
3997 sub set_migration_caps {
3998     my ($vmid) = @_;
3999
4000     my $cap_ref = [];
4001
4002     my $enabled_cap = {
4003         "auto-converge" => 1,
4004         "xbzrle" => 1,
4005         "x-rdma-pin-all" => 0,
4006         "zero-blocks" => 0,
4007         "compress" => 0
4008     };
4009
4010     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4011
4012     for my $supported_capability (@$supported_capabilities) {
4013         push @$cap_ref, {
4014             capability => $supported_capability->{capability},
4015             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4016         };
4017     }
4018
4019     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4020 }
4021
4022 my $fast_plug_option = {
4023     'lock' => 1,
4024     'name' => 1,
4025     'onboot' => 1,
4026     'shares' => 1,
4027     'startup' => 1,
4028     'description' => 1,
4029     'protection' => 1,
4030 };
4031
4032 # hotplug changes in [PENDING]
4033 # $selection hash can be used to only apply specified options, for
4034 # example: { cores => 1 } (only apply changed 'cores')
4035 # $errors ref is used to return error messages
4036 sub vmconfig_hotplug_pending {
4037     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4038
4039     my $defaults = load_defaults();
4040
4041     # commit values which do not have any impact on running VM first
4042     # Note: those option cannot raise errors, we we do not care about
4043     # $selection and always apply them.
4044
4045     my $add_error = sub {
4046         my ($opt, $msg) = @_;
4047         $errors->{$opt} = "hotplug problem - $msg";
4048     };
4049
4050     my $changes = 0;
4051     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4052         if ($fast_plug_option->{$opt}) {
4053             $conf->{$opt} = $conf->{pending}->{$opt};
4054             delete $conf->{pending}->{$opt};
4055             $changes = 1;
4056         }
4057     }
4058
4059     if ($changes) {
4060         PVE::QemuConfig->write_config($vmid, $conf);
4061         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4062     }
4063
4064     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4065
4066     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4067     while (my ($opt, $force) = each %$pending_delete_hash) {
4068         next if $selection && !$selection->{$opt};
4069         eval {
4070             if ($opt eq 'hotplug') {
4071                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4072             } elsif ($opt eq 'tablet') {
4073                 die "skip\n" if !$hotplug_features->{usb};
4074                 if ($defaults->{tablet}) {
4075                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4076                 } else {
4077                     vm_deviceunplug($vmid, $conf, $opt);
4078                 }
4079             } elsif ($opt =~ m/^usb\d+/) {
4080                 die "skip\n";
4081                 # since we cannot reliably hot unplug usb devices
4082                 # we are disabling it
4083                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4084                 vm_deviceunplug($vmid, $conf, $opt);
4085             } elsif ($opt eq 'vcpus') {
4086                 die "skip\n" if !$hotplug_features->{cpu};
4087                 qemu_cpu_hotplug($vmid, $conf, undef);
4088             } elsif ($opt eq 'balloon') {
4089                 # enable balloon device is not hotpluggable
4090                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4091             } elsif ($fast_plug_option->{$opt}) {
4092                 # do nothing
4093             } elsif ($opt =~ m/^net(\d+)$/) {
4094                 die "skip\n" if !$hotplug_features->{network};
4095                 vm_deviceunplug($vmid, $conf, $opt);
4096             } elsif (is_valid_drivename($opt)) {
4097                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4098                 vm_deviceunplug($vmid, $conf, $opt);
4099                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4100             } elsif ($opt =~ m/^memory$/) {
4101                 die "skip\n" if !$hotplug_features->{memory};
4102                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4103             } elsif ($opt eq 'cpuunits') {
4104                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4105             } elsif ($opt eq 'cpulimit') {
4106                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4107             } else {
4108                 die "skip\n";
4109             }
4110         };
4111         if (my $err = $@) {
4112             &$add_error($opt, $err) if $err ne "skip\n";
4113         } else {
4114             # save new config if hotplug was successful
4115             delete $conf->{$opt};
4116             vmconfig_undelete_pending_option($conf, $opt);
4117             PVE::QemuConfig->write_config($vmid, $conf);
4118             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4119         }
4120     }
4121
4122     foreach my $opt (keys %{$conf->{pending}}) {
4123         next if $selection && !$selection->{$opt};
4124         my $value = $conf->{pending}->{$opt};
4125         eval {
4126             if ($opt eq 'hotplug') {
4127                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4128             } elsif ($opt eq 'tablet') {
4129                 die "skip\n" if !$hotplug_features->{usb};
4130                 if ($value == 1) {
4131                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4132                 } elsif ($value == 0) {
4133                     vm_deviceunplug($vmid, $conf, $opt);
4134                 }
4135             } elsif ($opt =~ m/^usb\d+$/) {
4136                 die "skip\n";
4137                 # since we cannot reliably hot unplug usb devices
4138                 # we are disabling it
4139                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4140                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4141                 die "skip\n" if !$d;
4142                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4143             } elsif ($opt eq 'vcpus') {
4144                 die "skip\n" if !$hotplug_features->{cpu};
4145                 qemu_cpu_hotplug($vmid, $conf, $value);
4146             } elsif ($opt eq 'balloon') {
4147                 # enable/disable balloning device is not hotpluggable
4148                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4149                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4150                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4151
4152                 # allow manual ballooning if shares is set to zero
4153                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4154                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4155                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4156                 }
4157             } elsif ($opt =~ m/^net(\d+)$/) {
4158                 # some changes can be done without hotplug
4159                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4160                                     $vmid, $opt, $value);
4161             } elsif (is_valid_drivename($opt)) {
4162                 # some changes can be done without hotplug
4163                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4164                                      $vmid, $opt, $value, 1);
4165             } elsif ($opt =~ m/^memory$/) { #dimms
4166                 die "skip\n" if !$hotplug_features->{memory};
4167                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4168             } elsif ($opt eq 'cpuunits') {
4169                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4170             } elsif ($opt eq 'cpulimit') {
4171                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4172                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4173             } else {
4174                 die "skip\n";  # skip non-hot-pluggable options
4175             }
4176         };
4177         if (my $err = $@) {
4178             &$add_error($opt, $err) if $err ne "skip\n";
4179         } else {
4180             # save new config if hotplug was successful
4181             $conf->{$opt} = $value;
4182             delete $conf->{pending}->{$opt};
4183             PVE::QemuConfig->write_config($vmid, $conf);
4184             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4185         }
4186     }
4187 }
4188
4189 sub try_deallocate_drive {
4190     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4191
4192     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4193         my $volid = $drive->{file};
4194         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4195             my $sid = PVE::Storage::parse_volume_id($volid);
4196             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4197
4198             # check if the disk is really unused
4199             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4200                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4201             PVE::Storage::vdisk_free($storecfg, $volid);
4202             return 1;
4203         } else {
4204             # If vm is not owner of this disk remove from config
4205             return 1;
4206         }
4207     }
4208
4209     return undef;
4210 }
4211
4212 sub vmconfig_delete_or_detach_drive {
4213     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4214
4215     my $drive = parse_drive($opt, $conf->{$opt});
4216
4217     my $rpcenv = PVE::RPCEnvironment::get();
4218     my $authuser = $rpcenv->get_user();
4219
4220     if ($force) {
4221         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4222         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4223     } else {
4224         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4225     }
4226 }
4227
4228 sub vmconfig_apply_pending {
4229     my ($vmid, $conf, $storecfg) = @_;
4230
4231     # cold plug
4232
4233     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4234     while (my ($opt, $force) = each %$pending_delete_hash) {
4235         die "internal error" if $opt =~ m/^unused/;
4236         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4237         if (!defined($conf->{$opt})) {
4238             vmconfig_undelete_pending_option($conf, $opt);
4239             PVE::QemuConfig->write_config($vmid, $conf);
4240         } elsif (is_valid_drivename($opt)) {
4241             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4242             vmconfig_undelete_pending_option($conf, $opt);
4243             delete $conf->{$opt};
4244             PVE::QemuConfig->write_config($vmid, $conf);
4245         } else {
4246             vmconfig_undelete_pending_option($conf, $opt);
4247             delete $conf->{$opt};
4248             PVE::QemuConfig->write_config($vmid, $conf);
4249         }
4250     }
4251
4252     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4253
4254     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4255         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4256
4257         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4258             # skip if nothing changed
4259         } elsif (is_valid_drivename($opt)) {
4260             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4261                 if defined($conf->{$opt});
4262             $conf->{$opt} = $conf->{pending}->{$opt};
4263         } else {
4264             $conf->{$opt} = $conf->{pending}->{$opt};
4265         }
4266
4267         delete $conf->{pending}->{$opt};
4268         PVE::QemuConfig->write_config($vmid, $conf);
4269     }
4270 }
4271
4272 my $safe_num_ne = sub {
4273     my ($a, $b) = @_;
4274
4275     return 0 if !defined($a) && !defined($b);
4276     return 1 if !defined($a);
4277     return 1 if !defined($b);
4278
4279     return $a != $b;
4280 };
4281
4282 my $safe_string_ne = sub {
4283     my ($a, $b) = @_;
4284
4285     return 0 if !defined($a) && !defined($b);
4286     return 1 if !defined($a);
4287     return 1 if !defined($b);
4288
4289     return $a ne $b;
4290 };
4291
4292 sub vmconfig_update_net {
4293     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4294
4295     my $newnet = parse_net($value);
4296
4297     if ($conf->{$opt}) {
4298         my $oldnet = parse_net($conf->{$opt});
4299
4300         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4301             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4302             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4303             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4304
4305             # for non online change, we try to hot-unplug
4306             die "skip\n" if !$hotplug;
4307             vm_deviceunplug($vmid, $conf, $opt);
4308         } else {
4309
4310             die "internal error" if $opt !~ m/net(\d+)/;
4311             my $iface = "tap${vmid}i$1";
4312
4313             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4314                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4315                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4316                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4317                 PVE::Network::tap_unplug($iface);
4318                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4319             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4320                 # Rate can be applied on its own but any change above needs to
4321                 # include the rate in tap_plug since OVS resets everything.
4322                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4323             }
4324
4325             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4326                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4327             }
4328
4329             return 1;
4330         }
4331     }
4332
4333     if ($hotplug) {
4334         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4335     } else {
4336         die "skip\n";
4337     }
4338 }
4339
4340 sub vmconfig_update_disk {
4341     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4342
4343     # fixme: do we need force?
4344
4345     my $drive = parse_drive($opt, $value);
4346
4347     if ($conf->{$opt}) {
4348
4349         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4350
4351             my $media = $drive->{media} || 'disk';
4352             my $oldmedia = $old_drive->{media} || 'disk';
4353             die "unable to change media type\n" if $media ne $oldmedia;
4354
4355             if (!drive_is_cdrom($old_drive)) {
4356
4357                 if ($drive->{file} ne $old_drive->{file}) {
4358
4359                     die "skip\n" if !$hotplug;
4360
4361                     # unplug and register as unused
4362                     vm_deviceunplug($vmid, $conf, $opt);
4363                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4364
4365                 } else {
4366                     # update existing disk
4367
4368                     # skip non hotpluggable value
4369                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4370                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4371                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4372                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4373                         die "skip\n";
4374                     }
4375
4376                     # apply throttle
4377                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4378                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4379                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4380                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4381                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4382                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4383                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4384                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4385                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4386                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4387                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4388                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4389
4390                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4391                                                    ($drive->{mbps} || 0)*1024*1024,
4392                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4393                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4394                                                    $drive->{iops} || 0,
4395                                                    $drive->{iops_rd} || 0,
4396                                                    $drive->{iops_wr} || 0,
4397                                                    ($drive->{mbps_max} || 0)*1024*1024,
4398                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4399                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4400                                                    $drive->{iops_max} || 0,
4401                                                    $drive->{iops_rd_max} || 0,
4402                                                    $drive->{iops_wr_max} || 0);
4403
4404                     }
4405
4406                     return 1;
4407                 }
4408
4409             } else { # cdrom
4410
4411                 if ($drive->{file} eq 'none') {
4412                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4413                 } else {
4414                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4415                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4416                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4417                 }
4418
4419                 return 1;
4420             }
4421         }
4422     }
4423
4424     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4425     # hotplug new disks
4426     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4427     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4428 }
4429
4430 sub vm_start {
4431     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4432         $forcemachine, $spice_ticket) = @_;
4433
4434     PVE::QemuConfig->lock_config($vmid, sub {
4435         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4436
4437         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4438
4439         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4440
4441         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4442
4443         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4444             vmconfig_apply_pending($vmid, $conf, $storecfg);
4445             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4446         }
4447
4448         my $defaults = load_defaults();
4449
4450         # set environment variable useful inside network script
4451         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4452
4453         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4454
4455         my $migrate_port = 0;
4456         my $migrate_uri;
4457         if ($statefile) {
4458             if ($statefile eq 'tcp') {
4459                 my $localip = "localhost";
4460                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4461                 my $nodename = PVE::INotify::nodename();
4462                 if ($datacenterconf->{migration_unsecure}) {
4463                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4464                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4465                 }
4466                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4467                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4468                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4469                 push @$cmd, '-incoming', $migrate_uri;
4470                 push @$cmd, '-S';
4471
4472             } elsif ($statefile eq 'unix') {
4473                 # should be default for secure migrations as a ssh TCP forward
4474                 # tunnel is not deterministic reliable ready and fails regurarly
4475                 # to set up in time, so use UNIX socket forwards
4476                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4477                 unlink $socket_addr;
4478
4479                 $migrate_uri = "unix:$socket_addr";
4480
4481                 push @$cmd, '-incoming', $migrate_uri;
4482                 push @$cmd, '-S';
4483
4484             } else {
4485                 push @$cmd, '-loadstate', $statefile;
4486             }
4487         } elsif ($paused) {
4488             push @$cmd, '-S';
4489         }
4490
4491         # host pci devices
4492         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4493           my $d = parse_hostpci($conf->{"hostpci$i"});
4494           next if !$d;
4495           my $pcidevices = $d->{pciid};
4496           foreach my $pcidevice (@$pcidevices) {
4497                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4498
4499                 my $info = pci_device_info("0000:$pciid");
4500                 die "IOMMU not present\n" if !check_iommu_support();
4501                 die "no pci device info for device '$pciid'\n" if !$info;
4502                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4503                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4504           }
4505         }
4506
4507         PVE::Storage::activate_volumes($storecfg, $vollist);
4508
4509         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4510             my $cmd = [];
4511             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4512             eval  { run_command($cmd); };
4513         }
4514
4515         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4516                                                   : $defaults->{cpuunits};
4517
4518         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4519
4520         my %properties = (
4521             Slice => 'qemu.slice',
4522             KillMode => 'none',
4523             CPUShares => $cpuunits
4524         );
4525
4526         if (my $cpulimit = $conf->{cpulimit}) {
4527             $properties{CPUQuota} = int($cpulimit * 100);
4528         }
4529         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4530
4531         if ($conf->{hugepages}) {
4532
4533             my $code = sub {
4534                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4535                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4536
4537                 PVE::QemuServer::Memory::hugepages_mount();
4538                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4539
4540                 eval  {
4541                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4542                     run_command($cmd, %run_params);
4543                 };
4544
4545                 if (my $err = $@) {
4546                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4547                     die $err;
4548                 }
4549
4550                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4551             };
4552             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4553
4554         } else {
4555             eval  {
4556                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4557                 run_command($cmd, %run_params);
4558             };
4559         }
4560
4561         if (my $err = $@) {
4562             # deactivate volumes if start fails
4563             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4564             die "start failed: $err";
4565         }
4566
4567         print "migration listens on $migrate_uri\n" if $migrate_uri;
4568
4569         if ($statefile && $statefile ne 'tcp')  {
4570             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4571             warn $@ if $@;
4572         }
4573
4574         if ($migratedfrom) {
4575
4576             eval {
4577                 set_migration_caps($vmid);
4578             };
4579             warn $@ if $@;
4580
4581             if ($spice_port) {
4582                 print "spice listens on port $spice_port\n";
4583                 if ($spice_ticket) {
4584                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4585                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4586                 }
4587             }
4588
4589         } else {
4590
4591             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4592                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4593                     if $conf->{balloon};
4594             }
4595
4596             foreach my $opt (keys %$conf) {
4597                 next if $opt !~  m/^net\d+$/;
4598                 my $nicconf = parse_net($conf->{$opt});
4599                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4600             }
4601         }
4602
4603         vm_mon_cmd_nocheck($vmid, 'qom-set',
4604                     path => "machine/peripheral/balloon0",
4605                     property => "guest-stats-polling-interval",
4606                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4607
4608     });
4609 }
4610
4611 sub vm_mon_cmd {
4612     my ($vmid, $execute, %params) = @_;
4613
4614     my $cmd = { execute => $execute, arguments => \%params };
4615     vm_qmp_command($vmid, $cmd);
4616 }
4617
4618 sub vm_mon_cmd_nocheck {
4619     my ($vmid, $execute, %params) = @_;
4620
4621     my $cmd = { execute => $execute, arguments => \%params };
4622     vm_qmp_command($vmid, $cmd, 1);
4623 }
4624
4625 sub vm_qmp_command {
4626     my ($vmid, $cmd, $nocheck) = @_;
4627
4628     my $res;
4629
4630     my $timeout;
4631     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4632         $timeout = $cmd->{arguments}->{timeout};
4633         delete $cmd->{arguments}->{timeout};
4634     }
4635
4636     eval {
4637         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4638         my $sname = qmp_socket($vmid);
4639         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4640             my $qmpclient = PVE::QMPClient->new();
4641
4642             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4643         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4644             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4645                 if scalar(%{$cmd->{arguments}});
4646             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4647         } else {
4648             die "unable to open monitor socket\n";
4649         }
4650     };
4651     if (my $err = $@) {
4652         syslog("err", "VM $vmid qmp command failed - $err");
4653         die $err;
4654     }
4655
4656     return $res;
4657 }
4658
4659 sub vm_human_monitor_command {
4660     my ($vmid, $cmdline) = @_;
4661
4662     my $res;
4663
4664     my $cmd = {
4665         execute => 'human-monitor-command',
4666         arguments => { 'command-line' => $cmdline},
4667     };
4668
4669     return vm_qmp_command($vmid, $cmd);
4670 }
4671
4672 sub vm_commandline {
4673     my ($storecfg, $vmid) = @_;
4674
4675     my $conf = PVE::QemuConfig->load_config($vmid);
4676
4677     my $defaults = load_defaults();
4678
4679     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4680
4681     return PVE::Tools::cmd2string($cmd);
4682 }
4683
4684 sub vm_reset {
4685     my ($vmid, $skiplock) = @_;
4686
4687     PVE::QemuConfig->lock_config($vmid, sub {
4688
4689         my $conf = PVE::QemuConfig->load_config($vmid);
4690
4691         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4692
4693         vm_mon_cmd($vmid, "system_reset");
4694     });
4695 }
4696
4697 sub get_vm_volumes {
4698     my ($conf) = @_;
4699
4700     my $vollist = [];
4701     foreach_volid($conf, sub {
4702         my ($volid, $is_cdrom) = @_;
4703
4704         return if $volid =~ m|^/|;
4705
4706         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4707         return if !$sid;
4708
4709         push @$vollist, $volid;
4710     });
4711
4712     return $vollist;
4713 }
4714
4715 sub vm_stop_cleanup {
4716     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4717
4718     eval {
4719
4720         if (!$keepActive) {
4721             my $vollist = get_vm_volumes($conf);
4722             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4723         }
4724
4725         foreach my $ext (qw(mon qmp pid vnc qga)) {
4726             unlink "/var/run/qemu-server/${vmid}.$ext";
4727         }
4728
4729         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4730     };
4731     warn $@ if $@; # avoid errors - just warn
4732 }
4733
4734 # Note: use $nockeck to skip tests if VM configuration file exists.
4735 # We need that when migration VMs to other nodes (files already moved)
4736 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4737 sub vm_stop {
4738     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4739
4740     $force = 1 if !defined($force) && !$shutdown;
4741
4742     if ($migratedfrom){
4743         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4744         kill 15, $pid if $pid;
4745         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4746         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4747         return;
4748     }
4749
4750     PVE::QemuConfig->lock_config($vmid, sub {
4751
4752         my $pid = check_running($vmid, $nocheck);
4753         return if !$pid;
4754
4755         my $conf;
4756         if (!$nocheck) {
4757             $conf = PVE::QemuConfig->load_config($vmid);
4758             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4759             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4760                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4761                 $timeout = $opts->{down} if $opts->{down};
4762             }
4763         }
4764
4765         $timeout = 60 if !defined($timeout);
4766
4767         eval {
4768             if ($shutdown) {
4769                 if (defined($conf) && $conf->{agent}) {
4770                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4771                 } else {
4772                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4773                 }
4774             } else {
4775                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4776             }
4777         };
4778         my $err = $@;
4779
4780         if (!$err) {
4781             my $count = 0;
4782             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4783                 $count++;
4784                 sleep 1;
4785             }
4786
4787             if ($count >= $timeout) {
4788                 if ($force) {
4789                     warn "VM still running - terminating now with SIGTERM\n";
4790                     kill 15, $pid;
4791                 } else {
4792                     die "VM quit/powerdown failed - got timeout\n";
4793                 }
4794             } else {
4795                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4796                 return;
4797             }
4798         } else {
4799             if ($force) {
4800                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4801                 kill 15, $pid;
4802             } else {
4803                 die "VM quit/powerdown failed\n";
4804             }
4805         }
4806
4807         # wait again
4808         $timeout = 10;
4809
4810         my $count = 0;
4811         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4812             $count++;
4813             sleep 1;
4814         }
4815
4816         if ($count >= $timeout) {
4817             warn "VM still running - terminating now with SIGKILL\n";
4818             kill 9, $pid;
4819             sleep 1;
4820         }
4821
4822         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4823    });
4824 }
4825
4826 sub vm_suspend {
4827     my ($vmid, $skiplock) = @_;
4828
4829     PVE::QemuConfig->lock_config($vmid, sub {
4830
4831         my $conf = PVE::QemuConfig->load_config($vmid);
4832
4833         PVE::QemuConfig->check_lock($conf)
4834             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4835
4836         vm_mon_cmd($vmid, "stop");
4837     });
4838 }
4839
4840 sub vm_resume {
4841     my ($vmid, $skiplock, $nocheck) = @_;
4842
4843     PVE::QemuConfig->lock_config($vmid, sub {
4844
4845         if (!$nocheck) {
4846
4847             my $conf = PVE::QemuConfig->load_config($vmid);
4848
4849             PVE::QemuConfig->check_lock($conf)
4850                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4851
4852             vm_mon_cmd($vmid, "cont");
4853
4854         } else {
4855             vm_mon_cmd_nocheck($vmid, "cont");
4856         }
4857     });
4858 }
4859
4860 sub vm_sendkey {
4861     my ($vmid, $skiplock, $key) = @_;
4862
4863     PVE::QemuConfig->lock_config($vmid, sub {
4864
4865         my $conf = PVE::QemuConfig->load_config($vmid);
4866
4867         # there is no qmp command, so we use the human monitor command
4868         vm_human_monitor_command($vmid, "sendkey $key");
4869     });
4870 }
4871
4872 sub vm_destroy {
4873     my ($storecfg, $vmid, $skiplock) = @_;
4874
4875     PVE::QemuConfig->lock_config($vmid, sub {
4876
4877         my $conf = PVE::QemuConfig->load_config($vmid);
4878
4879         if (!check_running($vmid)) {
4880             destroy_vm($storecfg, $vmid, undef, $skiplock);
4881         } else {
4882             die "VM $vmid is running - destroy failed\n";
4883         }
4884     });
4885 }
4886
4887 # pci helpers
4888
4889 sub file_write {
4890     my ($filename, $buf) = @_;
4891
4892     my $fh = IO::File->new($filename, "w");
4893     return undef if !$fh;
4894
4895     my $res = print $fh $buf;
4896
4897     $fh->close();
4898
4899     return $res;
4900 }
4901
4902 sub pci_device_info {
4903     my ($name) = @_;
4904
4905     my $res;
4906
4907     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4908     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4909
4910     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4911     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4912
4913     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4914     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4915
4916     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4917     return undef if !defined($product) || $product !~ s/^0x//;
4918
4919     $res = {
4920         name => $name,
4921         vendor => $vendor,
4922         product => $product,
4923         domain => $domain,
4924         bus => $bus,
4925         slot => $slot,
4926         func => $func,
4927         irq => $irq,
4928         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4929     };
4930
4931     return $res;
4932 }
4933
4934 sub pci_dev_reset {
4935     my ($dev) = @_;
4936
4937     my $name = $dev->{name};
4938
4939     my $fn = "$pcisysfs/devices/$name/reset";
4940
4941     return file_write($fn, "1");
4942 }
4943
4944 sub pci_dev_bind_to_vfio {
4945     my ($dev) = @_;
4946
4947     my $name = $dev->{name};
4948
4949     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4950
4951     if (!-d $vfio_basedir) {
4952         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4953     }
4954     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4955
4956     my $testdir = "$vfio_basedir/$name";
4957     return 1 if -d $testdir;
4958
4959     my $data = "$dev->{vendor} $dev->{product}";
4960     return undef if !file_write("$vfio_basedir/new_id", $data);
4961
4962     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4963     if (!file_write($fn, $name)) {
4964         return undef if -f $fn;
4965     }
4966
4967     $fn = "$vfio_basedir/bind";
4968     if (! -d $testdir) {
4969         return undef if !file_write($fn, $name);
4970     }
4971
4972     return -d $testdir;
4973 }
4974
4975 sub pci_dev_group_bind_to_vfio {
4976     my ($pciid) = @_;
4977
4978     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4979
4980     if (!-d $vfio_basedir) {
4981         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4982     }
4983     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4984
4985     # get IOMMU group devices
4986     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4987       my @devs = grep /^0000:/, readdir($D);
4988     closedir($D);
4989
4990     foreach my $pciid (@devs) {
4991         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4992
4993         # pci bridges, switches or root ports are not supported
4994         # they have a pci_bus subdirectory so skip them
4995         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4996
4997         my $info = pci_device_info($1);
4998         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4999     }
5000
5001     return 1;
5002 }
5003
5004 # vzdump restore implementaion
5005
5006 sub tar_archive_read_firstfile {
5007     my $archive = shift;
5008
5009     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5010
5011     # try to detect archive type first
5012     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5013         die "unable to open file '$archive'\n";
5014     my $firstfile = <$fh>;
5015     kill 15, $pid;
5016     close $fh;
5017
5018     die "ERROR: archive contaions no data\n" if !$firstfile;
5019     chomp $firstfile;
5020
5021     return $firstfile;
5022 }
5023
5024 sub tar_restore_cleanup {
5025     my ($storecfg, $statfile) = @_;
5026
5027     print STDERR "starting cleanup\n";
5028
5029     if (my $fd = IO::File->new($statfile, "r")) {
5030         while (defined(my $line = <$fd>)) {
5031             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5032                 my $volid = $2;
5033                 eval {
5034                     if ($volid =~ m|^/|) {
5035                         unlink $volid || die 'unlink failed\n';
5036                     } else {
5037                         PVE::Storage::vdisk_free($storecfg, $volid);
5038                     }
5039                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5040                 };
5041                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5042             } else {
5043                 print STDERR "unable to parse line in statfile - $line";
5044             }
5045         }
5046         $fd->close();
5047     }
5048 }
5049
5050 sub restore_archive {
5051     my ($archive, $vmid, $user, $opts) = @_;
5052
5053     my $format = $opts->{format};
5054     my $comp;
5055
5056     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5057         $format = 'tar' if !$format;
5058         $comp = 'gzip';
5059     } elsif ($archive =~ m/\.tar$/) {
5060         $format = 'tar' if !$format;
5061     } elsif ($archive =~ m/.tar.lzo$/) {
5062         $format = 'tar' if !$format;
5063         $comp = 'lzop';
5064     } elsif ($archive =~ m/\.vma$/) {
5065         $format = 'vma' if !$format;
5066     } elsif ($archive =~ m/\.vma\.gz$/) {
5067         $format = 'vma' if !$format;
5068         $comp = 'gzip';
5069     } elsif ($archive =~ m/\.vma\.lzo$/) {
5070         $format = 'vma' if !$format;
5071         $comp = 'lzop';
5072     } else {
5073         $format = 'vma' if !$format; # default
5074     }
5075
5076     # try to detect archive format
5077     if ($format eq 'tar') {
5078         return restore_tar_archive($archive, $vmid, $user, $opts);
5079     } else {
5080         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5081     }
5082 }
5083
5084 sub restore_update_config_line {
5085     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5086
5087     return if $line =~ m/^\#qmdump\#/;
5088     return if $line =~ m/^\#vzdump\#/;
5089     return if $line =~ m/^lock:/;
5090     return if $line =~ m/^unused\d+:/;
5091     return if $line =~ m/^parent:/;
5092     return if $line =~ m/^template:/; # restored VM is never a template
5093
5094     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5095     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5096         # try to convert old 1.X settings
5097         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5098         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5099             my ($model, $macaddr) = split(/\=/, $devconfig);
5100             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5101             my $net = {
5102                 model => $model,
5103                 bridge => "vmbr$ind",
5104                 macaddr => $macaddr,
5105             };
5106             my $netstr = print_net($net);
5107
5108             print $outfd "net$cookie->{netcount}: $netstr\n";
5109             $cookie->{netcount}++;
5110         }
5111     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5112         my ($id, $netstr) = ($1, $2);
5113         my $net = parse_net($netstr);
5114         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5115         $netstr = print_net($net);
5116         print $outfd "$id: $netstr\n";
5117     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5118         my $virtdev = $1;
5119         my $value = $3;
5120         my $di = parse_drive($virtdev, $value);
5121         if (defined($di->{backup}) && !$di->{backup}) {
5122             print $outfd "#$line";
5123         } elsif ($map->{$virtdev}) {
5124             delete $di->{format}; # format can change on restore
5125             $di->{file} = $map->{$virtdev};
5126             $value = print_drive($vmid, $di);
5127             print $outfd "$virtdev: $value\n";
5128         } else {
5129             print $outfd $line;
5130         }
5131     } else {
5132         print $outfd $line;
5133     }
5134 }
5135
5136 sub scan_volids {
5137     my ($cfg, $vmid) = @_;
5138
5139     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5140
5141     my $volid_hash = {};
5142     foreach my $storeid (keys %$info) {
5143         foreach my $item (@{$info->{$storeid}}) {
5144             next if !($item->{volid} && $item->{size});
5145             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5146             $volid_hash->{$item->{volid}} = $item;
5147         }
5148     }
5149
5150     return $volid_hash;
5151 }
5152
5153 sub is_volume_in_use {
5154     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5155
5156     my $path = PVE::Storage::path($storecfg, $volid);
5157
5158     my $scan_config = sub {
5159         my ($cref, $snapname) = @_;
5160
5161         foreach my $key (keys %$cref) {
5162             my $value = $cref->{$key};
5163             if (is_valid_drivename($key)) {
5164                 next if $skip_drive && $key eq $skip_drive;
5165                 my $drive = parse_drive($key, $value);
5166                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5167                 return 1 if $volid eq $drive->{file};
5168                 if ($drive->{file} =~ m!^/!) {
5169                     return 1 if $drive->{file} eq $path;
5170                 } else {
5171                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5172                     next if !$storeid;
5173                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5174                     next if !$scfg;
5175                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5176                 }
5177             }
5178         }
5179
5180         return 0;
5181     };
5182
5183     return 1 if &$scan_config($conf);
5184
5185     undef $skip_drive;
5186
5187     foreach my $snapname (keys %{$conf->{snapshots}}) {
5188         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5189     }
5190
5191     return 0;
5192 }
5193
5194 sub update_disksize {
5195     my ($vmid, $conf, $volid_hash) = @_;
5196
5197     my $changes;
5198
5199     my $used = {};
5200
5201     # Note: it is allowed to define multiple storages with same path (alias), so
5202     # we need to check both 'volid' and real 'path' (two different volid can point
5203     # to the same path).
5204
5205     my $usedpath = {};
5206
5207     # update size info
5208     foreach my $opt (keys %$conf) {
5209         if (is_valid_drivename($opt)) {
5210             my $drive = parse_drive($opt, $conf->{$opt});
5211             my $volid = $drive->{file};
5212             next if !$volid;
5213
5214             $used->{$volid} = 1;
5215             if ($volid_hash->{$volid} &&
5216                 (my $path = $volid_hash->{$volid}->{path})) {
5217                 $usedpath->{$path} = 1;
5218             }
5219
5220             next if drive_is_cdrom($drive);
5221             next if !$volid_hash->{$volid};
5222
5223             $drive->{size} = $volid_hash->{$volid}->{size};
5224             my $new = print_drive($vmid, $drive);
5225             if ($new ne $conf->{$opt}) {
5226                 $changes = 1;
5227                 $conf->{$opt} = $new;
5228             }
5229         }
5230     }
5231
5232     # remove 'unusedX' entry if volume is used
5233     foreach my $opt (keys %$conf) {
5234         next if $opt !~ m/^unused\d+$/;
5235         my $volid = $conf->{$opt};
5236         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5237         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5238             $changes = 1;
5239             delete $conf->{$opt};
5240         }
5241     }
5242
5243     foreach my $volid (sort keys %$volid_hash) {
5244         next if $volid =~ m/vm-$vmid-state-/;
5245         next if $used->{$volid};
5246         my $path = $volid_hash->{$volid}->{path};
5247         next if !$path; # just to be sure
5248         next if $usedpath->{$path};
5249         $changes = 1;
5250         PVE::QemuConfig->add_unused_volume($conf, $volid);
5251         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5252     }
5253
5254     return $changes;
5255 }
5256
5257 sub rescan {
5258     my ($vmid, $nolock) = @_;
5259
5260     my $cfg = PVE::Storage::config();
5261
5262     my $volid_hash = scan_volids($cfg, $vmid);
5263
5264     my $updatefn =  sub {
5265         my ($vmid) = @_;
5266
5267         my $conf = PVE::QemuConfig->load_config($vmid);
5268
5269         PVE::QemuConfig->check_lock($conf);
5270
5271         my $vm_volids = {};
5272         foreach my $volid (keys %$volid_hash) {
5273             my $info = $volid_hash->{$volid};
5274             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5275         }
5276
5277         my $changes = update_disksize($vmid, $conf, $vm_volids);
5278
5279         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5280     };
5281
5282     if (defined($vmid)) {
5283         if ($nolock) {
5284             &$updatefn($vmid);
5285         } else {
5286             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5287         }
5288     } else {
5289         my $vmlist = config_list();
5290         foreach my $vmid (keys %$vmlist) {
5291             if ($nolock) {
5292                 &$updatefn($vmid);
5293             } else {
5294                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5295             }
5296         }
5297     }
5298 }
5299
5300 sub restore_vma_archive {
5301     my ($archive, $vmid, $user, $opts, $comp) = @_;
5302
5303     my $input = $archive eq '-' ? "<&STDIN" : undef;
5304     my $readfrom = $archive;
5305
5306     my $uncomp = '';
5307     if ($comp) {
5308         $readfrom = '-';
5309         my $qarchive = PVE::Tools::shellquote($archive);
5310         if ($comp eq 'gzip') {
5311             $uncomp = "zcat $qarchive|";
5312         } elsif ($comp eq 'lzop') {
5313             $uncomp = "lzop -d -c $qarchive|";
5314         } else {
5315             die "unknown compression method '$comp'\n";
5316         }
5317
5318     }
5319
5320     my $tmpdir = "/var/tmp/vzdumptmp$$";
5321     rmtree $tmpdir;
5322
5323     # disable interrupts (always do cleanups)
5324     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5325         warn "got interrupt - ignored\n";
5326     };
5327
5328     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5329     POSIX::mkfifo($mapfifo, 0600);
5330     my $fifofh;
5331
5332     my $openfifo = sub {
5333         open($fifofh, '>', $mapfifo) || die $!;
5334     };
5335
5336     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5337
5338     my $oldtimeout;
5339     my $timeout = 5;
5340
5341     my $devinfo = {};
5342
5343     my $rpcenv = PVE::RPCEnvironment::get();
5344
5345     my $conffile = PVE::QemuConfig->config_file($vmid);
5346     my $tmpfn = "$conffile.$$.tmp";
5347
5348     # Note: $oldconf is undef if VM does not exists
5349     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5350     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5351
5352     my $print_devmap = sub {
5353         my $virtdev_hash = {};
5354
5355         my $cfgfn = "$tmpdir/qemu-server.conf";
5356
5357         # we can read the config - that is already extracted
5358         my $fh = IO::File->new($cfgfn, "r") ||
5359             "unable to read qemu-server.conf - $!\n";
5360
5361         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5362         if (-f $fwcfgfn) {
5363             my $pve_firewall_dir = '/etc/pve/firewall';
5364             mkdir $pve_firewall_dir; # make sure the dir exists
5365             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5366         }
5367
5368         while (defined(my $line = <$fh>)) {
5369             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5370                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5371                 die "archive does not contain data for drive '$virtdev'\n"
5372                     if !$devinfo->{$devname};
5373                 if (defined($opts->{storage})) {
5374                     $storeid = $opts->{storage} || 'local';
5375                 } elsif (!$storeid) {
5376                     $storeid = 'local';
5377                 }
5378                 $format = 'raw' if !$format;
5379                 $devinfo->{$devname}->{devname} = $devname;
5380                 $devinfo->{$devname}->{virtdev} = $virtdev;
5381                 $devinfo->{$devname}->{format} = $format;
5382                 $devinfo->{$devname}->{storeid} = $storeid;
5383
5384                 # check permission on storage
5385                 my $pool = $opts->{pool}; # todo: do we need that?
5386                 if ($user ne 'root@pam') {
5387                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5388                 }
5389
5390                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5391             }
5392         }
5393
5394         foreach my $devname (keys %$devinfo) {
5395             die "found no device mapping information for device '$devname'\n"
5396                 if !$devinfo->{$devname}->{virtdev};
5397         }
5398
5399         my $cfg = PVE::Storage::config();
5400
5401         # create empty/temp config
5402         if ($oldconf) {
5403             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5404             foreach_drive($oldconf, sub {
5405                 my ($ds, $drive) = @_;
5406
5407                 return if drive_is_cdrom($drive);
5408
5409                 my $volid = $drive->{file};
5410
5411                 return if !$volid || $volid =~ m|^/|;
5412
5413                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5414                 return if !$path || !$owner || ($owner != $vmid);
5415
5416                 # Note: only delete disk we want to restore
5417                 # other volumes will become unused
5418                 if ($virtdev_hash->{$ds}) {
5419                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5420                     if (my $err = $@) {
5421                         warn $err;
5422                     }
5423                 }
5424             });
5425
5426             # delete vmstate files
5427             # since after the restore we have no snapshots anymore
5428             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5429                 my $snap = $oldconf->{snapshots}->{$snapname};
5430                 if ($snap->{vmstate}) {
5431                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5432                     if (my $err = $@) {
5433                         warn $err;
5434                     }
5435                 }
5436             }
5437         }
5438
5439         my $map = {};
5440         foreach my $virtdev (sort keys %$virtdev_hash) {
5441             my $d = $virtdev_hash->{$virtdev};
5442             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5443             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5444
5445             # test if requested format is supported
5446             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5447             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5448             $d->{format} = $defFormat if !$supported;
5449
5450             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5451                                                   $d->{format}, undef, $alloc_size);
5452             print STDERR "new volume ID is '$volid'\n";
5453             $d->{volid} = $volid;
5454             my $path = PVE::Storage::path($cfg, $volid);
5455
5456             PVE::Storage::activate_volumes($cfg,[$volid]);
5457
5458             my $write_zeros = 1;
5459             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5460                 $write_zeros = 0;
5461             }
5462
5463             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5464
5465             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5466             $map->{$virtdev} = $volid;
5467         }
5468
5469         $fh->seek(0, 0) || die "seek failed - $!\n";
5470
5471         my $outfd = new IO::File ($tmpfn, "w") ||
5472             die "unable to write config for VM $vmid\n";
5473
5474         my $cookie = { netcount => 0 };
5475         while (defined(my $line = <$fh>)) {
5476             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5477         }
5478
5479         $fh->close();
5480         $outfd->close();
5481     };
5482
5483     eval {
5484         # enable interrupts
5485         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5486             die "interrupted by signal\n";
5487         };
5488         local $SIG{ALRM} = sub { die "got timeout\n"; };
5489
5490         $oldtimeout = alarm($timeout);
5491
5492         my $parser = sub {
5493             my $line = shift;
5494
5495             print "$line\n";
5496
5497             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5498                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5499                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5500             } elsif ($line =~ m/^CTIME: /) {
5501                 # we correctly received the vma config, so we can disable
5502                 # the timeout now for disk allocation (set to 10 minutes, so
5503                 # that we always timeout if something goes wrong)
5504                 alarm(600);
5505                 &$print_devmap();
5506                 print $fifofh "done\n";
5507                 my $tmp = $oldtimeout || 0;
5508                 $oldtimeout = undef;
5509                 alarm($tmp);
5510                 close($fifofh);
5511             }
5512         };
5513
5514         print "restore vma archive: $cmd\n";
5515         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5516     };
5517     my $err = $@;
5518
5519     alarm($oldtimeout) if $oldtimeout;
5520
5521     my $vollist = [];
5522     foreach my $devname (keys %$devinfo) {
5523         my $volid = $devinfo->{$devname}->{volid};
5524         push @$vollist, $volid if $volid;
5525     }
5526
5527     my $cfg = PVE::Storage::config();
5528     PVE::Storage::deactivate_volumes($cfg, $vollist);
5529
5530     unlink $mapfifo;
5531
5532     if ($err) {
5533         rmtree $tmpdir;
5534         unlink $tmpfn;
5535
5536         foreach my $devname (keys %$devinfo) {
5537             my $volid = $devinfo->{$devname}->{volid};
5538             next if !$volid;
5539             eval {
5540                 if ($volid =~ m|^/|) {
5541                     unlink $volid || die 'unlink failed\n';
5542                 } else {
5543                     PVE::Storage::vdisk_free($cfg, $volid);
5544                 }
5545                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5546             };
5547             print STDERR "unable to cleanup '$volid' - $@" if $@;
5548         }
5549         die $err;
5550     }
5551
5552     rmtree $tmpdir;
5553
5554     rename($tmpfn, $conffile) ||
5555         die "unable to commit configuration file '$conffile'\n";
5556
5557     PVE::Cluster::cfs_update(); # make sure we read new file
5558
5559     eval { rescan($vmid, 1); };
5560     warn $@ if $@;
5561 }
5562
5563 sub restore_tar_archive {
5564     my ($archive, $vmid, $user, $opts) = @_;
5565
5566     if ($archive ne '-') {
5567         my $firstfile = tar_archive_read_firstfile($archive);
5568         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5569             if $firstfile ne 'qemu-server.conf';
5570     }
5571
5572     my $storecfg = PVE::Storage::config();
5573
5574     # destroy existing data - keep empty config
5575     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5576     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5577
5578     my $tocmd = "/usr/lib/qemu-server/qmextract";
5579
5580     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5581     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5582     $tocmd .= ' --prealloc' if $opts->{prealloc};
5583     $tocmd .= ' --info' if $opts->{info};
5584
5585     # tar option "xf" does not autodetect compression when read from STDIN,
5586     # so we pipe to zcat
5587     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5588         PVE::Tools::shellquote("--to-command=$tocmd");
5589
5590     my $tmpdir = "/var/tmp/vzdumptmp$$";
5591     mkpath $tmpdir;
5592
5593     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5594     local $ENV{VZDUMP_VMID} = $vmid;
5595     local $ENV{VZDUMP_USER} = $user;
5596
5597     my $conffile = PVE::QemuConfig->config_file($vmid);
5598     my $tmpfn = "$conffile.$$.tmp";
5599
5600     # disable interrupts (always do cleanups)
5601     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5602         print STDERR "got interrupt - ignored\n";
5603     };
5604
5605     eval {
5606         # enable interrupts
5607         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5608             die "interrupted by signal\n";
5609         };
5610
5611         if ($archive eq '-') {
5612             print "extracting archive from STDIN\n";
5613             run_command($cmd, input => "<&STDIN");
5614         } else {
5615             print "extracting archive '$archive'\n";
5616             run_command($cmd);
5617         }
5618
5619         return if $opts->{info};
5620
5621         # read new mapping
5622         my $map = {};
5623         my $statfile = "$tmpdir/qmrestore.stat";
5624         if (my $fd = IO::File->new($statfile, "r")) {
5625             while (defined (my $line = <$fd>)) {
5626                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5627                     $map->{$1} = $2 if $1;
5628                 } else {
5629                     print STDERR "unable to parse line in statfile - $line\n";
5630                 }
5631             }
5632             $fd->close();
5633         }
5634
5635         my $confsrc = "$tmpdir/qemu-server.conf";
5636
5637         my $srcfd = new IO::File($confsrc, "r") ||
5638             die "unable to open file '$confsrc'\n";
5639
5640         my $outfd = new IO::File ($tmpfn, "w") ||
5641             die "unable to write config for VM $vmid\n";
5642
5643         my $cookie = { netcount => 0 };
5644         while (defined (my $line = <$srcfd>)) {
5645             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5646         }
5647
5648         $srcfd->close();
5649         $outfd->close();
5650     };
5651     my $err = $@;
5652
5653     if ($err) {
5654
5655         unlink $tmpfn;
5656
5657         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5658
5659         die $err;
5660     }
5661
5662     rmtree $tmpdir;
5663
5664     rename $tmpfn, $conffile ||
5665         die "unable to commit configuration file '$conffile'\n";
5666
5667     PVE::Cluster::cfs_update(); # make sure we read new file
5668
5669     eval { rescan($vmid, 1); };
5670     warn $@ if $@;
5671 };
5672
5673 sub foreach_writable_storage {
5674     my ($conf, $func) = @_;
5675
5676     my $sidhash = {};
5677
5678     foreach my $ds (keys %$conf) {
5679         next if !is_valid_drivename($ds);
5680
5681         my $drive = parse_drive($ds, $conf->{$ds});
5682         next if !$drive;
5683         next if drive_is_cdrom($drive);
5684
5685         my $volid = $drive->{file};
5686
5687         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5688         $sidhash->{$sid} = $sid if $sid;
5689     }
5690
5691     foreach my $sid (sort keys %$sidhash) {
5692         &$func($sid);
5693     }
5694 }
5695
5696 sub do_snapshots_with_qemu {
5697     my ($storecfg, $volid) = @_;
5698
5699     my $storage_name = PVE::Storage::parse_volume_id($volid);
5700
5701     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5702         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5703         return 1;
5704     }
5705
5706     if ($volid =~ m/\.(qcow2|qed)$/){
5707         return 1;
5708     }
5709
5710     return undef;
5711 }
5712
5713 sub qga_check_running {
5714     my ($vmid) = @_;
5715
5716     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5717     if ($@) {
5718         warn "Qemu Guest Agent are not running - $@";
5719         return 0;
5720     }
5721     return 1;
5722 }
5723
5724 sub template_create {
5725     my ($vmid, $conf, $disk) = @_;
5726
5727     my $storecfg = PVE::Storage::config();
5728
5729     foreach_drive($conf, sub {
5730         my ($ds, $drive) = @_;
5731
5732         return if drive_is_cdrom($drive);
5733         return if $disk && $ds ne $disk;
5734
5735         my $volid = $drive->{file};
5736         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5737
5738         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5739         $drive->{file} = $voliddst;
5740         $conf->{$ds} = print_drive($vmid, $drive);
5741         PVE::QemuConfig->write_config($vmid, $conf);
5742     });
5743 }
5744
5745 sub qemu_img_convert {
5746     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5747
5748     my $storecfg = PVE::Storage::config();
5749     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5750     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5751
5752     if ($src_storeid && $dst_storeid) {
5753
5754         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5755
5756         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5757         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5758
5759         my $src_format = qemu_img_format($src_scfg, $src_volname);
5760         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5761
5762         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5763         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5764
5765         my $cmd = [];
5766         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5767         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5768         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5769         if ($is_zero_initialized) {
5770             push @$cmd, "zeroinit:$dst_path";
5771         } else {
5772             push @$cmd, $dst_path;
5773         }
5774
5775         my $parser = sub {
5776             my $line = shift;
5777             if($line =~ m/\((\S+)\/100\%\)/){
5778                 my $percent = $1;
5779                 my $transferred = int($size * $percent / 100);
5780                 my $remaining = $size - $transferred;
5781
5782                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5783             }
5784
5785         };
5786
5787         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5788         my $err = $@;
5789         die "copy failed: $err" if $err;
5790     }
5791 }
5792
5793 sub qemu_img_format {
5794     my ($scfg, $volname) = @_;
5795
5796     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5797         return $1;
5798     } else {
5799         return "raw";
5800     }
5801 }
5802
5803 sub qemu_drive_mirror {
5804     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5805
5806     my $storecfg = PVE::Storage::config();
5807     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5808
5809     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5810
5811     my $format = qemu_img_format($dst_scfg, $dst_volname);
5812
5813     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5814
5815     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5816
5817     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5818     $opts->{format} = $format if $format;
5819
5820     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5821
5822     my $finish_job = sub {
5823         while (1) {
5824             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5825             my $stat = @$stats[0];
5826             last if !$stat;
5827             sleep 1;
5828         }
5829     };
5830
5831     eval {
5832     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5833         while (1) {
5834             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5835             my $stat = @$stats[0];
5836             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5837             die "error job is not mirroring" if $stat->{type} ne "mirror";
5838
5839             my $busy = $stat->{busy};
5840             my $ready = $stat->{ready};
5841
5842             if (my $total = $stat->{len}) {
5843                 my $transferred = $stat->{offset} || 0;
5844                 my $remaining = $total - $transferred;
5845                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5846
5847                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5848             }
5849
5850
5851             if ($stat->{ready} eq 'true') {
5852
5853                 last if $vmiddst != $vmid;
5854
5855                 # try to switch the disk if source and destination are on the same guest
5856                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5857                 if (!$@) {
5858                     &$finish_job();
5859                     last;
5860                 }
5861                 die $@ if $@ !~ m/cannot be completed/;
5862             }
5863             sleep 1;
5864         }
5865
5866
5867     };
5868     my $err = $@;
5869
5870     my $cancel_job = sub {
5871         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5872         &$finish_job();
5873     };
5874
5875     if ($err) {
5876         eval { &$cancel_job(); };
5877         die "mirroring error: $err";
5878     }
5879
5880     if ($vmiddst != $vmid) {
5881         # if we clone a disk for a new target vm, we don't switch the disk
5882         &$cancel_job(); # so we call block-job-cancel
5883     }
5884 }
5885
5886 sub clone_disk {
5887     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5888         $newvmid, $storage, $format, $full, $newvollist) = @_;
5889
5890     my $newvolid;
5891
5892     if (!$full) {
5893         print "create linked clone of drive $drivename ($drive->{file})\n";
5894         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5895         push @$newvollist, $newvolid;
5896     } else {
5897         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5898         $storeid = $storage if $storage;
5899
5900         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5901         if (!$format) {
5902             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5903             $format = qemu_img_format($scfg, $volname);
5904         }
5905
5906         # test if requested format is supported - else use default
5907         my $supported = grep { $_ eq $format } @$validFormats;
5908         $format = $defFormat if !$supported;
5909
5910         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5911
5912         print "create full clone of drive $drivename ($drive->{file})\n";
5913         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5914         push @$newvollist, $newvolid;
5915
5916         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
5917
5918         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5919         if (!$running || $snapname) {
5920             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5921         } else {
5922
5923             my $kvmver = get_running_qemu_version ($vmid);
5924             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
5925                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
5926                     if $drive->{iothread};
5927             }
5928
5929             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5930         }
5931     }
5932
5933     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5934
5935     my $disk = $drive;
5936     $disk->{format} = undef;
5937     $disk->{file} = $newvolid;
5938     $disk->{size} = $size;
5939
5940     return $disk;
5941 }
5942
5943 # this only works if VM is running
5944 sub get_current_qemu_machine {
5945     my ($vmid) = @_;
5946
5947     my $cmd = { execute => 'query-machines', arguments => {} };
5948     my $res = vm_qmp_command($vmid, $cmd);
5949
5950     my ($current, $default);
5951     foreach my $e (@$res) {
5952         $default = $e->{name} if $e->{'is-default'};
5953         $current = $e->{name} if $e->{'is-current'};
5954     }
5955
5956     # fallback to the default machine if current is not supported by qemu
5957     return $current || $default || 'pc';
5958 }
5959
5960 sub get_running_qemu_version {
5961     my ($vmid) = @_;
5962     my $cmd = { execute => 'query-version', arguments => {} };
5963     my $res = vm_qmp_command($vmid, $cmd);
5964     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
5965 }
5966
5967 sub qemu_machine_feature_enabled {
5968     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5969
5970     my $current_major;
5971     my $current_minor;
5972
5973     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5974
5975         $current_major = $3;
5976         $current_minor = $4;
5977
5978     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5979
5980         $current_major = $1;
5981         $current_minor = $2;
5982     }
5983
5984     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5985
5986
5987 }
5988
5989 sub qemu_machine_pxe {
5990     my ($vmid, $conf, $machine) = @_;
5991
5992     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5993
5994     foreach my $opt (keys %$conf) {
5995         next if $opt !~ m/^net(\d+)$/;
5996         my $net = PVE::QemuServer::parse_net($conf->{$opt});
5997         next if !$net;
5998         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
5999         return $machine.".pxe" if $romfile =~ m/pxe/;
6000         last;
6001     }
6002
6003     return $machine;
6004 }
6005
6006 sub qemu_use_old_bios_files {
6007     my ($machine_type) = @_;
6008
6009     return if !$machine_type;
6010
6011     my $use_old_bios_files = undef;
6012
6013     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6014         $machine_type = $1;
6015         $use_old_bios_files = 1;
6016     } else {
6017         my $kvmver = kvm_user_version();
6018         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6019         # load new efi bios files on migration. So this hack is required to allow
6020         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6021         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6022         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6023     }
6024
6025     return ($use_old_bios_files, $machine_type);
6026 }
6027
6028 sub lspci {
6029
6030     my $devices = {};
6031
6032     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6033             my (undef, $id, $function) = @_;
6034             my $res = { id => $id, function => $function};
6035             push @{$devices->{$id}}, $res;
6036     });
6037
6038     return $devices;
6039 }
6040
6041 sub vm_iothreads_list {
6042     my ($vmid) = @_;
6043
6044     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6045
6046     my $iothreads = {};
6047     foreach my $iothread (@$res) {
6048         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6049     }
6050
6051     return $iothreads;
6052 }
6053
6054 sub scsihw_infos {
6055     my ($conf, $drive) = @_;
6056
6057     my $maxdev = 0;
6058
6059     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6060         $maxdev = 7;
6061     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6062         $maxdev = 1;
6063     } else {
6064         $maxdev = 256;
6065     }
6066
6067     my $controller = int($drive->{index} / $maxdev);
6068     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6069
6070     return ($maxdev, $controller, $controller_prefix);
6071 }
6072
6073 # bash completion helper
6074
6075 sub complete_backup_archives {
6076     my ($cmdname, $pname, $cvalue) = @_;
6077
6078     my $cfg = PVE::Storage::config();
6079
6080     my $storeid;
6081
6082     if ($cvalue =~ m/^([^:]+):/) {
6083         $storeid = $1;
6084     }
6085
6086     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6087
6088     my $res = [];
6089     foreach my $id (keys %$data) {
6090         foreach my $item (@{$data->{$id}}) {
6091             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6092             push @$res, $item->{volid} if defined($item->{volid});
6093         }
6094     }
6095
6096     return $res;
6097 }
6098
6099 my $complete_vmid_full = sub {
6100     my ($running) = @_;
6101
6102     my $idlist = vmstatus();
6103
6104     my $res = [];
6105
6106     foreach my $id (keys %$idlist) {
6107         my $d = $idlist->{$id};
6108         if (defined($running)) {
6109             next if $d->{template};
6110             next if $running && $d->{status} ne 'running';
6111             next if !$running && $d->{status} eq 'running';
6112         }
6113         push @$res, $id;
6114
6115     }
6116     return $res;
6117 };
6118
6119 sub complete_vmid {
6120     return &$complete_vmid_full();
6121 }
6122
6123 sub complete_vmid_stopped {
6124     return &$complete_vmid_full(0);
6125 }
6126
6127 sub complete_vmid_running {
6128     return &$complete_vmid_full(1);
6129 }
6130
6131 sub complete_storage {
6132
6133     my $cfg = PVE::Storage::config();
6134     my $ids = $cfg->{ids};
6135
6136     my $res = [];
6137     foreach my $sid (keys %$ids) {
6138         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6139         next if !$ids->{$sid}->{content}->{images};
6140         push @$res, $sid;
6141     }
6142
6143     return $res;
6144 }
6145
6146 1;