PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120
 121     # AMD CPUs
 122     athlon => 'AuthenticAMD',
 123     phenom  => 'AuthenticAMD',
 124     Opteron_G1  => 'AuthenticAMD',
 125     Opteron_G2  => 'AuthenticAMD',
 126     Opteron_G3  => 'AuthenticAMD',
 127     Opteron_G4  => 'AuthenticAMD',
 128     Opteron_G5  => 'AuthenticAMD',
 129
 130     # generic types, use vendor from host node
 131     host => 'default',
 132     kvm32 => 'default',
 133     kvm64 => 'default',
 134     qemu32 => 'default',
 135     qemu64 => 'default',
 136 };
 137
 138 my $cpu_fmt = {
 139     cputype => {
 140         description => "Emulated CPU type.",
 141         type => 'string',
 142         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 143         format_description => 'cputype',
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 218         minimum => 0,
 219         maximum => 500000,
 220         default => 1000,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012
 285 l24;; Linux 2.4 Kernel
 286 l26;; Linux 2.6/3.X Kernel
 287 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 288 EODESC
 289     },
 290     boot => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 294         pattern => '[acdn]{1,4}',
 295         default => 'cdn',
 296     },
 297     bootdisk => {
 298         optional => 1,
 299         type => 'string', format => 'pve-qm-bootdisk',
 300         description => "Enable booting from specified disk.",
 301         pattern => '(ide|sata|scsi|virtio)\d+',
 302     },
 303     smp => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPUs. Please use option -sockets instead.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     sockets => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPU sockets.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     cores => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of cores per socket.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     numa => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable NUMA.",
 328         default => 0,
 329     },
 330     hugepages => {
 331         optional => 1,
 332         type => 'string',
 333         description => "Enable/disable hugepages memory.",
 334         enum => [qw(any 2 1024)],
 335     },
 336     vcpus => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Number of hotplugged vcpus.",
 340         minimum => 1,
 341         default => 0,
 342     },
 343     acpi => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable ACPI.",
 347         default => 1,
 348     },
 349     agent => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable Qemu GuestAgent.",
 353         default => 0,
 354     },
 355     kvm => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable KVM hardware virtualization.",
 359         default => 1,
 360     },
 361     tdf => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable time drift fix.",
 365         default => 0,
 366     },
 367     localtime => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 371     },
 372     freeze => {
 373         optional => 1,
 374         type => 'boolean',
 375         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 376     },
 377     vga => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Select the VGA type.",
 381         verbose_description => "Select the VGA type. If you want to use high resolution" .
 382             " modes (>= 1280x1024x16) then you should use the options " .
 383             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 384             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 385             "display sever. For win* OS you can select how many independent " .
 386             "displays you want, Linux guests can add displays them self. " .
 387             "You can also run without any graphic card, using a serial device" .
 388             " as terminal.",
 389         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 390     },
 391     watchdog => {
 392         optional => 1,
 393         type => 'string', format => 'pve-qm-watchdog',
 394         description => "Create a virtual hardware watchdog device.",
 395         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 396             " (by a guest action), the watchdog must be periodically polled " .
 397             "by an agent inside the guest or else the watchdog will reset " .
 398             "the guest (or execute the respective action specified)",
 399     },
 400     startdate => {
 401         optional => 1,
 402         type => 'string',
 403         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 404         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 405         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 406         default => 'now',
 407     },
 408     startup =>  get_standard_option('pve-startup-order'),
 409     template => {
 410         optional => 1,
 411         type => 'boolean',
 412         description => "Enable/disable Template.",
 413         default => 0,
 414     },
 415     args => {
 416         optional => 1,
 417         type => 'string',
 418         description => "Arbitrary arguments passed to kvm.",
 419         verbose_description => <<EODESCR,
 420 Arbitrary arguments passed to kvm, for example:
 421
 422 args: -no-reboot -no-hpet
 423
 424 NOTE: this option is for experts only.
 425 EODESCR
 426     },
 427     tablet => {
 428         optional => 1,
 429         type => 'boolean',
 430         default => 1,
 431         description => "Enable/disable the USB tablet device.",
 432         verbose_description => "Enable/disable the USB tablet device. This device is " .
 433             "usually needed to allow absolute mouse positioning with VNC. " .
 434             "Else the mouse runs out of sync with normal VNC clients. " .
 435             "If you're running lots of console-only guests on one host, " .
 436             "you may consider disabling this to save some context switches. " .
 437             "This is turned off by default if you use spice (-vga=qxl).",
 438     },
 439     migrate_speed => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 443         minimum => 0,
 444         default => 0,
 445     },
 446     migrate_downtime => {
 447         optional => 1,
 448         type => 'number',
 449         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 450         minimum => 0,
 451         default => 0.1,
 452     },
 453     cdrom => {
 454         optional => 1,
 455         type => 'string', format => 'pve-qm-drive',
 456         typetext => 'volume',
 457         description => "This is an alias for option -ide2",
 458     },
 459     cpu => {
 460         optional => 1,
 461         description => "Emulated CPU type.",
 462         type => 'string',
 463         format => $cpu_fmt,
 464     },
 465     parent => get_standard_option('pve-snapshot-name', {
 466         optional => 1,
 467         description => "Parent snapshot name. This is used internally, and should not be modified.",
 468     }),
 469     snaptime => {
 470         optional => 1,
 471         description => "Timestamp for snapshots.",
 472         type => 'integer',
 473         minimum => 0,
 474     },
 475     vmstate => {
 476         optional => 1,
 477         type => 'string', format => 'pve-volume-id',
 478         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 479     },
 480     machine => {
 481         description => "Specific the Qemu machine type.",
 482         type => 'string',
 483         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 484         maxLength => 40,
 485         optional => 1,
 486     },
 487     smbios1 => {
 488         description => "Specify SMBIOS type 1 fields.",
 489         type => 'string', format => 'pve-qm-smbios1',
 490         maxLength => 256,
 491         optional => 1,
 492     },
 493     protection => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 497         default => 0,
 498     },
 499     bios => {
 500         optional => 1,
 501         type => 'string',
 502         enum => [ qw(seabios ovmf) ],
 503         description => "Select BIOS implementation.",
 504         default => 'seabios',
 505     },
 506 };
 507
 508 # what about other qemu settings ?
 509 #cpu => 'string',
 510 #machine => 'string',
 511 #fda => 'file',
 512 #fdb => 'file',
 513 #mtdblock => 'file',
 514 #sd => 'file',
 515 #pflash => 'file',
 516 #snapshot => 'bool',
 517 #bootp => 'file',
 518 ##tftp => 'dir',
 519 ##smb => 'dir',
 520 #kernel => 'file',
 521 #append => 'string',
 522 #initrd => 'file',
 523 ##soundhw => 'string',
 524
 525 while (my ($k, $v) = each %$confdesc) {
 526     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 527 }
 528
 529 my $MAX_IDE_DISKS = 4;
 530 my $MAX_SCSI_DISKS = 14;
 531 my $MAX_VIRTIO_DISKS = 16;
 532 my $MAX_SATA_DISKS = 6;
 533 my $MAX_USB_DEVICES = 5;
 534 my $MAX_NETS = 32;
 535 my $MAX_UNUSED_DISKS = 8;
 536 my $MAX_HOSTPCI_DEVICES = 4;
 537 my $MAX_SERIAL_PORTS = 4;
 538 my $MAX_PARALLEL_PORTS = 3;
 539 my $MAX_NUMA = 8;
 540
 541 my $numa_fmt = {
 542     cpus => {
 543         type => "string",
 544         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 545         description => "CPUs accessing this NUMA node.",
 546         format_description => "id[-id];...",
 547     },
 548     memory => {
 549         type => "number",
 550         description => "Amount of memory this NUMA node provides.",
 551         optional => 1,
 552     },
 553     hostnodes => {
 554         type => "string",
 555         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 556         description => "Host NUMA nodes to use.",
 557         format_description => "id[-id];...",
 558         optional => 1,
 559     },
 560     policy => {
 561         type => 'string',
 562         enum => [qw(preferred bind interleave)],
 563         description => "NUMA allocation policy.",
 564         optional => 1,
 565     },
 566 };
 567 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 568 my $numadesc = {
 569     optional => 1,
 570     type => 'string', format => $numa_fmt,
 571     description => "NUMA topology.",
 572 };
 573 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 574
 575 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 576     $confdesc->{"numa$i"} = $numadesc;
 577 }
 578
 579 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 580                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 581                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 582 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 583
 584 my $net_fmt_bridge_descr = <<__EOD__;
 585 Bridge to attach the network device to. The Proxmox VE standard bridge
 586 is called 'vmbr0'.
 587
 588 If you do not specify a bridge, we create a kvm user (NATed) network
 589 device, which provides DHCP and DNS services. The following addresses
 590 are used:
 591
 592  10.0.2.2   Gateway
 593  10.0.2.3   DNS Server
 594  10.0.2.4   SMB Server
 595
 596 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 597 __EOD__
 598
 599 my $net_fmt = {
 600     macaddr => {
 601         type => 'string',
 602         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 603         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 604         format_description => "XX:XX:XX:XX:XX:XX",
 605         optional => 1,
 606     },
 607     model => {
 608         type => 'string',
 609         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 610         format_description => 'model',
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => {
 738         type => 'string',
 739         format_description => 'image format',
 740         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 741         description => "The drive's backing file's data format.",
 742         optional => 1,
 743     },
 744     size => {
 745         type => 'string',
 746         format => 'disk-size',
 747         format_description => 'DiskSize',
 748         description => "Disk size. This is purely informational and has no effect.",
 749         optional => 1,
 750     },
 751     backup => {
 752         type => 'boolean',
 753         description => "Whether the drive should be included when making backups.",
 754         optional => 1,
 755     },
 756     werror => {
 757         type => 'string',
 758         enum => [qw(enospc ignore report stop)],
 759         description => 'Write error action.',
 760         optional => 1,
 761     },
 762     aio => {
 763         type => 'string',
 764         enum => [qw(native threads)],
 765         description => 'AIO type to use.',
 766         optional => 1,
 767     },
 768     discard => {
 769         type => 'string',
 770         enum => [qw(ignore on)],
 771         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 772         optional => 1,
 773     },
 774     detect_zeroes => {
 775         type => 'boolean',
 776         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 777         optional => 1,
 778     },
 779     serial => {
 780         type => 'string',
 781         format => 'urlencoded',
 782         format_description => 'serial',
 783         maxLength => 20*3, # *3 since it's %xx url enoded
 784         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 785         optional => 1,
 786     }
 787 );
 788
 789 my %rerror_fmt = (
 790     rerror => {
 791         type => 'string',
 792         enum => [qw(ignore report stop)],
 793         description => 'Read error action.',
 794         optional => 1,
 795     },
 796 );
 797
 798 my %iothread_fmt = ( iothread => {
 799         type => 'boolean',
 800         description => "Whether to use iothreads for this drive",
 801         optional => 1,
 802 });
 803
 804 my %model_fmt = (
 805     model => {
 806         type => 'string',
 807         format => 'urlencoded',
 808         format_description => 'model',
 809         maxLength => 40*3, # *3 since it's %xx url enoded
 810         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 811         optional => 1,
 812     },
 813 );
 814
 815 my %queues_fmt = (
 816     queues => {
 817         type => 'integer',
 818         description => "Number of queues.",
 819         minimum => 2,
 820         optional => 1
 821     }
 822 );
 823
 824 my $add_throttle_desc = sub {
 825     my ($key, $type, $what, $unit, $longunit) = @_;
 826     $drivedesc_base{$key} = {
 827         type => $type,
 828         format_description => $unit,
 829         description => "Maximum $what speed in $longunit per second.",
 830         optional => 1,
 831     };
 832 };
 833 # throughput: (leaky bucket)
 834 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 835 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 836 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 837 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 838 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 839 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 840 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 841 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 842 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 843
 844 # pools: (pool of IO before throttling starts taking effect)
 845 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 846 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 847 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 848 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 849 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 850 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 851
 852 my $ide_fmt = {
 853     %drivedesc_base,
 854     %rerror_fmt,
 855     %model_fmt,
 856 };
 857
 858 my $idedesc = {
 859     optional => 1,
 860     type => 'string', format => $ide_fmt,
 861     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 862 };
 863 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 864
 865 my $scsi_fmt = {
 866     %drivedesc_base,
 867     %iothread_fmt,
 868     %queues_fmt,
 869 };
 870 my $scsidesc = {
 871     optional => 1,
 872     type => 'string', format => $scsi_fmt,
 873     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 874 };
 875 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 876
 877 my $sata_fmt = {
 878     %drivedesc_base,
 879     %rerror_fmt,
 880 };
 881 my $satadesc = {
 882     optional => 1,
 883     type => 'string', format => $sata_fmt,
 884     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 885 };
 886 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 887
 888 my $virtio_fmt = {
 889     %drivedesc_base,
 890     %iothread_fmt,
 891     %rerror_fmt,
 892 };
 893 my $virtiodesc = {
 894     optional => 1,
 895     type => 'string', format => $virtio_fmt,
 896     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 897 };
 898 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 899
 900 my $alldrive_fmt = {
 901     %drivedesc_base,
 902     %rerror_fmt,
 903     %iothread_fmt,
 904     %model_fmt,
 905     %queues_fmt,
 906 };
 907
 908 my $efidisk_fmt = {
 909     volume => { alias => 'file' },
 910     file => {
 911         type => 'string',
 912         format => 'pve-volume-id-or-qm-path',
 913         default_key => 1,
 914         format_description => 'volume',
 915         description => "The drive's backing volume.",
 916     },
 917     format => {
 918         type => 'string',
 919         format_description => 'image format',
 920         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 921         description => "The drive's backing file's data format.",
 922         optional => 1,
 923     },
 924     size => {
 925         type => 'string',
 926         format => 'disk-size',
 927         format_description => 'DiskSize',
 928         description => "Disk size. This is purely informational and has no effect.",
 929         optional => 1,
 930     },
 931 };
 932
 933 my $efidisk_desc = {
 934     optional => 1,
 935     type => 'string', format => $efidisk_fmt,
 936     description => "Configure a Disk for storing EFI vars",
 937 };
 938
 939 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 940
 941 my $usb_fmt = {
 942     host => {
 943         default_key => 1,
 944         type => 'string', format => 'pve-qm-usb-device',
 945         format_description => 'HOSTUSBDEVICE|spice',
 946         description => <<EODESCR,
 947 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 948
 949  'bus-port(.port)*' (decimal numbers) or
 950  'vendor_id:product_id' (hexadeciaml numbers) or
 951  'spice'
 952
 953 You can use the 'lsusb -t' command to list existing usb devices.
 954
 955 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 956
 957 The value 'spice' can be used to add a usb redirection devices for spice.
 958 EODESCR
 959     },
 960     usb3 => {
 961         optional => 1,
 962         type => 'boolean',
 963         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 964         default => 0,
 965     },
 966 };
 967
 968 my $usbdesc = {
 969     optional => 1,
 970     type => 'string', format => $usb_fmt,
 971     description => "Configure an USB device (n is 0 to 4).",
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 974
 975 # NOTE: the match-groups of this regex are used in parse_hostpci
 976 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 977 my $hostpci_fmt = {
 978     host => {
 979         default_key => 1,
 980         type => 'string',
 981         pattern => qr/$PCIRE(;$PCIRE)*/,
 982         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 983         description => <<EODESCR,
 984 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 985 of PCI virtual functions of the host. HOSTPCIID syntax is:
 986
 987 'bus:dev.func' (hexadecimal numbers)
 988
 989 You can us the 'lspci' command to list existing PCI devices.
 990 EODESCR
 991     },
 992     rombar => {
 993         type => 'boolean',
 994         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 995         optional => 1,
 996         default => 1,
 997     },
 998     pcie => {
 999         type => 'boolean',
1000         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1001         optional => 1,
1002         default => 0,
1003     },
1004     'x-vga' => {
1005         type => 'boolean',
1006         description =>  "Enable vfio-vga device support.",
1007         optional => 1,
1008         default => 0,
1009     },
1010 };
1011 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1012
1013 my $hostpcidesc = {
1014         optional => 1,
1015         type => 'string', format => 'pve-qm-hostpci',
1016         description => "Map host PCI devices into guest.",
1017         verbose_description =>  <<EODESCR,
1018 Map host PCI devices into guest.
1019
1020 NOTE: This option allows direct access to host hardware. So it is no longer
1021 possible to migrate such machines - use with special care.
1022
1023 CAUTION: Experimental! User reported problems with this option.
1024 EODESCR
1025 };
1026 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1027
1028 my $serialdesc = {
1029         optional => 1,
1030         type => 'string',
1031         pattern => '(/dev/.+|socket)',
1032         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1033         verbose_description =>  <<EODESCR,
1034 Create a serial device inside the VM (n is 0 to 3), and pass through a
1035 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1036 host side (use 'qm terminal' to open a terminal connection).
1037
1038 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1039
1040 CAUTION: Experimental! User reported problems with this option.
1041 EODESCR
1042 };
1043
1044 my $paralleldesc= {
1045         optional => 1,
1046         type => 'string',
1047         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1048         description =>  "Map host parallel devices (n is 0 to 2).",
1049         verbose_description =>  <<EODESCR,
1050 Map host parallel devices (n is 0 to 2).
1051
1052 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1053
1054 CAUTION: Experimental! User reported problems with this option.
1055 EODESCR
1056 };
1057
1058 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1059     $confdesc->{"parallel$i"} = $paralleldesc;
1060 }
1061
1062 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1063     $confdesc->{"serial$i"} = $serialdesc;
1064 }
1065
1066 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1067     $confdesc->{"hostpci$i"} = $hostpcidesc;
1068 }
1069
1070 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1071     $drivename_hash->{"ide$i"} = 1;
1072     $confdesc->{"ide$i"} = $idedesc;
1073 }
1074
1075 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1076     $drivename_hash->{"sata$i"} = 1;
1077     $confdesc->{"sata$i"} = $satadesc;
1078 }
1079
1080 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1081     $drivename_hash->{"scsi$i"} = 1;
1082     $confdesc->{"scsi$i"} = $scsidesc ;
1083 }
1084
1085 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1086     $drivename_hash->{"virtio$i"} = 1;
1087     $confdesc->{"virtio$i"} = $virtiodesc;
1088 }
1089
1090 $drivename_hash->{efidisk0} = 1;
1091 $confdesc->{efidisk0} = $efidisk_desc;
1092
1093 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1094     $confdesc->{"usb$i"} = $usbdesc;
1095 }
1096
1097 my $unuseddesc = {
1098     optional => 1,
1099     type => 'string', format => 'pve-volume-id',
1100     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1101 };
1102
1103 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1104     $confdesc->{"unused$i"} = $unuseddesc;
1105 }
1106
1107 my $kvm_api_version = 0;
1108
1109 sub kvm_version {
1110
1111     return $kvm_api_version if $kvm_api_version;
1112
1113     my $fh = IO::File->new("</dev/kvm") ||
1114         return 0;
1115
1116     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1117         $kvm_api_version = $v;
1118     }
1119
1120     $fh->close();
1121
1122     return  $kvm_api_version;
1123 }
1124
1125 my $kvm_user_version;
1126
1127 sub kvm_user_version {
1128
1129     return $kvm_user_version if $kvm_user_version;
1130
1131     $kvm_user_version = 'unknown';
1132
1133     my $code = sub {
1134         my $line = shift;
1135         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1136             $kvm_user_version = $2;
1137         }
1138     };
1139
1140     eval { run_command("kvm -version", outfunc => $code); };
1141     warn $@ if $@;
1142
1143     return $kvm_user_version;
1144
1145 }
1146
1147 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1148
1149 sub valid_drive_names {
1150     # order is important - used to autoselect boot disk
1151     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1152             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1153             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1154             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1155             'efidisk0');
1156 }
1157
1158 sub is_valid_drivename {
1159     my $dev = shift;
1160
1161     return defined($drivename_hash->{$dev});
1162 }
1163
1164 sub option_exists {
1165     my $key = shift;
1166     return defined($confdesc->{$key});
1167 }
1168
1169 sub nic_models {
1170     return $nic_model_list;
1171 }
1172
1173 sub os_list_description {
1174
1175     return {
1176         other => 'Other',
1177         wxp => 'Windows XP',
1178         w2k => 'Windows 2000',
1179         w2k3 =>, 'Windows 2003',
1180         w2k8 => 'Windows 2008',
1181         wvista => 'Windows Vista',
1182         win7 => 'Windows 7',
1183         win8 => 'Windows 8/2012',
1184         l24 => 'Linux 2.4',
1185         l26 => 'Linux 2.6',
1186     };
1187 }
1188
1189 my $cdrom_path;
1190
1191 sub get_cdrom_path {
1192
1193     return  $cdrom_path if $cdrom_path;
1194
1195     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1196     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1197     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1198 }
1199
1200 sub get_iso_path {
1201     my ($storecfg, $vmid, $cdrom) = @_;
1202
1203     if ($cdrom eq 'cdrom') {
1204         return get_cdrom_path();
1205     } elsif ($cdrom eq 'none') {
1206         return '';
1207     } elsif ($cdrom =~ m|^/|) {
1208         return $cdrom;
1209     } else {
1210         return PVE::Storage::path($storecfg, $cdrom);
1211     }
1212 }
1213
1214 # try to convert old style file names to volume IDs
1215 sub filename_to_volume_id {
1216     my ($vmid, $file, $media) = @_;
1217
1218     if (!($file eq 'none' || $file eq 'cdrom' ||
1219           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1220
1221         return undef if $file =~ m|/|;
1222
1223         if ($media && $media eq 'cdrom') {
1224             $file = "local:iso/$file";
1225         } else {
1226             $file = "local:$vmid/$file";
1227         }
1228     }
1229
1230     return $file;
1231 }
1232
1233 sub verify_media_type {
1234     my ($opt, $vtype, $media) = @_;
1235
1236     return if !$media;
1237
1238     my $etype;
1239     if ($media eq 'disk') {
1240         $etype = 'images';
1241     } elsif ($media eq 'cdrom') {
1242         $etype = 'iso';
1243     } else {
1244         die "internal error";
1245     }
1246
1247     return if ($vtype eq $etype);
1248
1249     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1250 }
1251
1252 sub cleanup_drive_path {
1253     my ($opt, $storecfg, $drive) = @_;
1254
1255     # try to convert filesystem paths to volume IDs
1256
1257     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1258         ($drive->{file} !~ m|^/dev/.+|) &&
1259         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1260         ($drive->{file} !~ m/^\d+$/)) {
1261         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1262         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1263         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1264         verify_media_type($opt, $vtype, $drive->{media});
1265         $drive->{file} = $volid;
1266     }
1267
1268     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1269 }
1270
1271 sub parse_hotplug_features {
1272     my ($data) = @_;
1273
1274     my $res = {};
1275
1276     return $res if $data eq '0';
1277
1278     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1279
1280     foreach my $feature (PVE::Tools::split_list($data)) {
1281         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1282             $res->{$1} = 1;
1283         } else {
1284             die "invalid hotplug feature '$feature'\n";
1285         }
1286     }
1287     return $res;
1288 }
1289
1290 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1291 sub pve_verify_hotplug_features {
1292     my ($value, $noerr) = @_;
1293
1294     return $value if parse_hotplug_features($value);
1295
1296     return undef if $noerr;
1297
1298     die "unable to parse hotplug option\n";
1299 }
1300
1301 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1302 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1303 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1304 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1305 #        [,iothread=on][,serial=serial][,model=model]
1306
1307 sub parse_drive {
1308     my ($key, $data) = @_;
1309
1310     my ($interface, $index);
1311
1312     if ($key =~ m/^([^\d]+)(\d+)$/) {
1313         $interface = $1;
1314         $index = $2;
1315     } else {
1316         return undef;
1317     }
1318
1319     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1320                                      : $confdesc->{$key}->{format};
1321     if (!$desc) {
1322         warn "invalid drive key: $key\n";
1323         return undef;
1324     }
1325     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1326     return undef if !$res;
1327     $res->{interface} = $interface;
1328     $res->{index} = $index;
1329
1330     my $error = 0;
1331     foreach my $opt (qw(bps bps_rd bps_wr)) {
1332         if (my $bps = defined(delete $res->{$opt})) {
1333             if (defined($res->{"m$opt"})) {
1334                 warn "both $opt and m$opt specified\n";
1335                 ++$error;
1336                 next;
1337             }
1338             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1339         }
1340     }
1341     return undef if $error;
1342
1343     return undef if $res->{mbps_rd} && $res->{mbps};
1344     return undef if $res->{mbps_wr} && $res->{mbps};
1345     return undef if $res->{iops_rd} && $res->{iops};
1346     return undef if $res->{iops_wr} && $res->{iops};
1347
1348     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1349         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1350         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1351         return undef if $res->{interface} eq 'virtio';
1352     }
1353
1354     if (my $size = $res->{size}) {
1355         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1356     }
1357
1358     return $res;
1359 }
1360
1361 sub print_drive {
1362     my ($vmid, $drive) = @_;
1363     my $data = { %$drive };
1364     delete $data->{$_} for qw(index interface);
1365     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1366 }
1367
1368 sub scsi_inquiry {
1369     my($fh, $noerr) = @_;
1370
1371     my $SG_IO = 0x2285;
1372     my $SG_GET_VERSION_NUM = 0x2282;
1373
1374     my $versionbuf = "\x00" x 8;
1375     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1376     if (!$ret) {
1377         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1378         return undef;
1379     }
1380     my $version = unpack("I", $versionbuf);
1381     if ($version < 30000) {
1382         die "scsi generic interface too old\n"  if !$noerr;
1383         return undef;
1384     }
1385
1386     my $buf = "\x00" x 36;
1387     my $sensebuf = "\x00" x 8;
1388     my $cmd = pack("C x3 C x1", 0x12, 36);
1389
1390     # see /usr/include/scsi/sg.h
1391     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1392
1393     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1394                       length($sensebuf), 0, length($buf), $buf,
1395                       $cmd, $sensebuf, 6000);
1396
1397     $ret = ioctl($fh, $SG_IO, $packet);
1398     if (!$ret) {
1399         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1400         return undef;
1401     }
1402
1403     my @res = unpack($sg_io_hdr_t, $packet);
1404     if ($res[17] || $res[18]) {
1405         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1406         return undef;
1407     }
1408
1409     my $res = {};
1410     (my $byte0, my $byte1, $res->{vendor},
1411      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1412
1413     $res->{removable} = $byte1 & 128 ? 1 : 0;
1414     $res->{type} = $byte0 & 31;
1415
1416     return $res;
1417 }
1418
1419 sub path_is_scsi {
1420     my ($path) = @_;
1421
1422     my $fh = IO::File->new("+<$path") || return undef;
1423     my $res = scsi_inquiry($fh, 1);
1424     close($fh);
1425
1426     return $res;
1427 }
1428
1429 sub machine_type_is_q35 {
1430     my ($conf) = @_;
1431
1432     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1433 }
1434
1435 sub print_tabletdevice_full {
1436     my ($conf) = @_;
1437
1438     my $q35 = machine_type_is_q35($conf);
1439
1440     # we use uhci for old VMs because tablet driver was buggy in older qemu
1441     my $usbbus = $q35 ? "ehci" : "uhci";
1442
1443     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1444 }
1445
1446 sub print_drivedevice_full {
1447     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1448
1449     my $device = '';
1450     my $maxdev = 0;
1451
1452     if ($drive->{interface} eq 'virtio') {
1453         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1454         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1455         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1456     } elsif ($drive->{interface} eq 'scsi') {
1457
1458         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1459         my $unit = $drive->{index} % $maxdev;
1460         my $devicetype = 'hd';
1461         my $path = '';
1462         if (drive_is_cdrom($drive)) {
1463             $devicetype = 'cd';
1464         } else {
1465             if ($drive->{file} =~ m|^/|) {
1466                 $path = $drive->{file};
1467                 if (my $info = path_is_scsi($path)) {
1468                     if ($info->{type} == 0) {
1469                         $devicetype = 'block';
1470                     } elsif ($info->{type} == 1) { # tape
1471                         $devicetype = 'generic';
1472                     }
1473                 }
1474             } else {
1475                  $path = PVE::Storage::path($storecfg, $drive->{file});
1476             }
1477
1478             if($path =~ m/^iscsi\:\/\//){
1479                 $devicetype = 'generic';
1480             }
1481         }
1482
1483         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1484             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1485         } else {
1486             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1487         }
1488
1489     } elsif ($drive->{interface} eq 'ide'){
1490         $maxdev = 2;
1491         my $controller = int($drive->{index} / $maxdev);
1492         my $unit = $drive->{index} % $maxdev;
1493         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1494
1495         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1496         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1497             $model = URI::Escape::uri_unescape($model);
1498             $device .= ",model=$model";
1499         }
1500     } elsif ($drive->{interface} eq 'sata'){
1501         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1502         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1503         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1504     } elsif ($drive->{interface} eq 'usb') {
1505         die "implement me";
1506         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1507     } else {
1508         die "unsupported interface type";
1509     }
1510
1511     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1512
1513     return $device;
1514 }
1515
1516 sub get_initiator_name {
1517     my $initiator;
1518
1519     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1520     while (defined(my $line = <$fh>)) {
1521         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1522         $initiator = $1;
1523         last;
1524     }
1525     $fh->close();
1526
1527     return $initiator;
1528 }
1529
1530 sub print_drive_full {
1531     my ($storecfg, $vmid, $drive) = @_;
1532
1533     my $path;
1534     my $volid = $drive->{file};
1535     my $format;
1536
1537     if (drive_is_cdrom($drive)) {
1538         $path = get_iso_path($storecfg, $vmid, $volid);
1539     } else {
1540         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1541         if ($storeid) {
1542             $path = PVE::Storage::path($storecfg, $volid);
1543             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1544             $format = qemu_img_format($scfg, $volname);
1545         } else {
1546             $path = $volid;
1547             $format = "raw";
1548         }
1549    }
1550
1551     my $opts = '';
1552     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1553     foreach my $o (@qemu_drive_options) {
1554         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1555     }
1556     if (my $serial = $drive->{serial}) {
1557         $serial = URI::Escape::uri_unescape($serial);
1558         $opts .= ",serial=$serial";
1559     }
1560
1561     $opts .= ",format=$format" if $format && !$drive->{format};
1562
1563     foreach my $o (qw(bps bps_rd bps_wr)) {
1564         my $v = $drive->{"m$o"};
1565         $opts .= ",$o=" . int($v*1024*1024) if $v;
1566     }
1567
1568     my $cache_direct = 0;
1569
1570     if (my $cache = $drive->{cache}) {
1571         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1572     } elsif (!drive_is_cdrom($drive)) {
1573         $opts .= ",cache=none";
1574         $cache_direct = 1;
1575     }
1576
1577     # aio native works only with O_DIRECT
1578     if (!$drive->{aio}) {
1579         if($cache_direct) {
1580             $opts .= ",aio=native";
1581         } else {
1582             $opts .= ",aio=threads";
1583         }
1584     }
1585
1586     if (!drive_is_cdrom($drive)) {
1587         my $detectzeroes;
1588         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1589             $detectzeroes = 'off';
1590         } elsif ($drive->{discard}) {
1591             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1592         } else {
1593             # This used to be our default with discard not being specified:
1594             $detectzeroes = 'on';
1595         }
1596         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1597     }
1598
1599     my $pathinfo = $path ? "file=$path," : '';
1600
1601     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1602 }
1603
1604 sub print_netdevice_full {
1605     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1606
1607     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1608
1609     my $device = $net->{model};
1610     if ($net->{model} eq 'virtio') {
1611          $device = 'virtio-net-pci';
1612      };
1613
1614     my $pciaddr = print_pci_addr("$netid", $bridges);
1615     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1616     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1617         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1618         my $vectors = $net->{queues} * 2 + 2;
1619         $tmpstr .= ",vectors=$vectors,mq=on";
1620     }
1621     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1622
1623     if ($use_old_bios_files) {
1624         my $romfile;
1625         if ($device eq 'virtio-net-pci') {
1626             $romfile = 'pxe-virtio.rom';
1627         } elsif ($device eq 'e1000') {
1628             $romfile = 'pxe-e1000.rom';
1629         } elsif ($device eq 'ne2k') {
1630             $romfile = 'pxe-ne2k_pci.rom';
1631         } elsif ($device eq 'pcnet') {
1632             $romfile = 'pxe-pcnet.rom';
1633         } elsif ($device eq 'rtl8139') {
1634             $romfile = 'pxe-rtl8139.rom';
1635         }
1636         $tmpstr .= ",romfile=$romfile" if $romfile;
1637     }
1638
1639     return $tmpstr;
1640 }
1641
1642 sub print_netdev_full {
1643     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1644
1645     my $i = '';
1646     if ($netid =~ m/^net(\d+)$/) {
1647         $i = int($1);
1648     }
1649
1650     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1651
1652     my $ifname = "tap${vmid}i$i";
1653
1654     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1655     die "interface name '$ifname' is too long (max 15 character)\n"
1656         if length($ifname) >= 16;
1657
1658     my $vhostparam = '';
1659     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1660
1661     my $vmname = $conf->{name} || "vm$vmid";
1662
1663     my $netdev = "";
1664     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1665
1666     if ($net->{bridge}) {
1667         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1668     } else {
1669         $netdev = "type=user,id=$netid,hostname=$vmname";
1670     }
1671
1672     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1673
1674     return $netdev;
1675 }
1676
1677
1678 sub print_cpu_device {
1679     my ($conf, $id) = @_;
1680
1681     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1682     my $cpu = $nokvm ? "qemu64" : "kvm64";
1683     if (my $cputype = $conf->{cpu}) {
1684         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1685             or die "Cannot parse cpu description: $cputype\n";
1686         $cpu = $cpuconf->{cputype};
1687     }
1688
1689     my $sockets = 1;
1690     $sockets = $conf->{sockets} if  $conf->{sockets};
1691     my $cores = $conf->{cores} || 1;
1692
1693     my $current_core = ($id - 1) % $cores;
1694     my $current_socket = int(($id - $current_core)/$cores);
1695
1696     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1697 }
1698
1699 sub drive_is_cdrom {
1700     my ($drive) = @_;
1701
1702     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1703
1704 }
1705
1706 sub parse_number_sets {
1707     my ($set) = @_;
1708     my $res = [];
1709     foreach my $part (split(/;/, $set)) {
1710         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1711             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1712             push @$res, [ $1, $2 ];
1713         } else {
1714             die "invalid range: $part\n";
1715         }
1716     }
1717     return $res;
1718 }
1719
1720 sub parse_numa {
1721     my ($data) = @_;
1722
1723     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1724     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1725     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1726     return $res;
1727 }
1728
1729 sub parse_hostpci {
1730     my ($value) = @_;
1731
1732     return undef if !$value;
1733
1734     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1735
1736     my @idlist = split(/;/, $res->{host});
1737     delete $res->{host};
1738     foreach my $id (@idlist) {
1739         if ($id =~ /^$PCIRE$/) {
1740             if (defined($2)) {
1741                 push @{$res->{pciid}}, { id => $1, function => $2 };
1742             } else {
1743                 my $pcidevices = lspci($1);
1744                 $res->{pciid} = $pcidevices->{$1};
1745             }
1746         } else {
1747             # should have been caught by parse_property_string already
1748             die "failed to parse PCI id: $id\n";
1749         }
1750     }
1751     return $res;
1752 }
1753
1754 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1755 sub parse_net {
1756     my ($data) = @_;
1757
1758     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1759     if ($@) {
1760         warn $@;
1761         return undef;
1762     }
1763     if (!defined($res->{macaddr})) {
1764         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1765         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1766     }
1767     return $res;
1768 }
1769
1770 sub print_net {
1771     my $net = shift;
1772
1773     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1774 }
1775
1776 sub add_random_macs {
1777     my ($settings) = @_;
1778
1779     foreach my $opt (keys %$settings) {
1780         next if $opt !~ m/^net(\d+)$/;
1781         my $net = parse_net($settings->{$opt});
1782         next if !$net;
1783         $settings->{$opt} = print_net($net);
1784     }
1785 }
1786
1787 sub vm_is_volid_owner {
1788     my ($storecfg, $vmid, $volid) = @_;
1789
1790     if ($volid !~  m|^/|) {
1791         my ($path, $owner);
1792         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1793         if ($owner && ($owner == $vmid)) {
1794             return 1;
1795         }
1796     }
1797
1798     return undef;
1799 }
1800
1801 sub split_flagged_list {
1802     my $text = shift || '';
1803     $text =~ s/[,;]/ /g;
1804     $text =~ s/^\s+//;
1805     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1806 }
1807
1808 sub join_flagged_list {
1809     my ($how, $lst) = @_;
1810     join $how, map { $lst->{$_} . $_ } keys %$lst;
1811 }
1812
1813 sub vmconfig_delete_pending_option {
1814     my ($conf, $key, $force) = @_;
1815
1816     delete $conf->{pending}->{$key};
1817     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1818     $pending_delete_hash->{$key} = $force ? '!' : '';
1819     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1820 }
1821
1822 sub vmconfig_undelete_pending_option {
1823     my ($conf, $key) = @_;
1824
1825     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1826     delete $pending_delete_hash->{$key};
1827
1828     if (%$pending_delete_hash) {
1829         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1830     } else {
1831         delete $conf->{pending}->{delete};
1832     }
1833 }
1834
1835 sub vmconfig_register_unused_drive {
1836     my ($storecfg, $vmid, $conf, $drive) = @_;
1837
1838     if (!drive_is_cdrom($drive)) {
1839         my $volid = $drive->{file};
1840         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1841             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1842         }
1843     }
1844 }
1845
1846 sub vmconfig_cleanup_pending {
1847     my ($conf) = @_;
1848
1849     # remove pending changes when nothing changed
1850     my $changes;
1851     foreach my $opt (keys %{$conf->{pending}}) {
1852         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1853             $changes = 1;
1854             delete $conf->{pending}->{$opt};
1855         }
1856     }
1857
1858     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1859     my $pending_delete_hash = {};
1860     while (my ($opt, $force) = each %$current_delete_hash) {
1861         if (defined($conf->{$opt})) {
1862             $pending_delete_hash->{$opt} = $force;
1863         } else {
1864             $changes = 1;
1865         }
1866     }
1867
1868     if (%$pending_delete_hash) {
1869         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1870     } else {
1871         delete $conf->{pending}->{delete};
1872     }
1873
1874     return $changes;
1875 }
1876
1877 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1878 my $smbios1_fmt = {
1879     uuid => {
1880         type => 'string',
1881         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1882         format_description => 'UUID',
1883         description => "Set SMBIOS1 UUID.",
1884         optional => 1,
1885     },
1886     version => {
1887         type => 'string',
1888         pattern => '\S+',
1889         format_description => 'string',
1890         description => "Set SMBIOS1 version.",
1891         optional => 1,
1892     },
1893     serial => {
1894         type => 'string',
1895         pattern => '\S+',
1896         format_description => 'string',
1897         description => "Set SMBIOS1 serial number.",
1898         optional => 1,
1899     },
1900     manufacturer => {
1901         type => 'string',
1902         pattern => '\S+',
1903         format_description => 'string',
1904         description => "Set SMBIOS1 manufacturer.",
1905         optional => 1,
1906     },
1907     product => {
1908         type => 'string',
1909         pattern => '\S+',
1910         format_description => 'string',
1911         description => "Set SMBIOS1 product ID.",
1912         optional => 1,
1913     },
1914     sku => {
1915         type => 'string',
1916         pattern => '\S+',
1917         format_description => 'string',
1918         description => "Set SMBIOS1 SKU string.",
1919         optional => 1,
1920     },
1921     family => {
1922         type => 'string',
1923         pattern => '\S+',
1924         format_description => 'string',
1925         description => "Set SMBIOS1 family string.",
1926         optional => 1,
1927     },
1928 };
1929
1930 sub parse_smbios1 {
1931     my ($data) = @_;
1932
1933     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1934     warn $@ if $@;
1935     return $res;
1936 }
1937
1938 sub print_smbios1 {
1939     my ($smbios1) = @_;
1940     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1941 }
1942
1943 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1944
1945 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1946 sub verify_bootdisk {
1947     my ($value, $noerr) = @_;
1948
1949     return $value if is_valid_drivename($value);
1950
1951     return undef if $noerr;
1952
1953     die "invalid boot disk '$value'\n";
1954 }
1955
1956 sub parse_watchdog {
1957     my ($value) = @_;
1958
1959     return undef if !$value;
1960
1961     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1962     warn $@ if $@;
1963     return $res;
1964 }
1965
1966 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1967 sub verify_usb_device {
1968     my ($value, $noerr) = @_;
1969
1970     return $value if parse_usb_device($value);
1971
1972     return undef if $noerr;
1973
1974     die "unable to parse usb device\n";
1975 }
1976
1977 # add JSON properties for create and set function
1978 sub json_config_properties {
1979     my $prop = shift;
1980
1981     foreach my $opt (keys %$confdesc) {
1982         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1983         $prop->{$opt} = $confdesc->{$opt};
1984     }
1985
1986     return $prop;
1987 }
1988
1989 sub check_type {
1990     my ($key, $value) = @_;
1991
1992     die "unknown setting '$key'\n" if !$confdesc->{$key};
1993
1994     my $type = $confdesc->{$key}->{type};
1995
1996     if (!defined($value)) {
1997         die "got undefined value\n";
1998     }
1999
2000     if ($value =~ m/[\n\r]/) {
2001         die "property contains a line feed\n";
2002     }
2003
2004     if ($type eq 'boolean') {
2005         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2006         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2007         die "type check ('boolean') failed - got '$value'\n";
2008     } elsif ($type eq 'integer') {
2009         return int($1) if $value =~ m/^(\d+)$/;
2010         die "type check ('integer') failed - got '$value'\n";
2011     } elsif ($type eq 'number') {
2012         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2013         die "type check ('number') failed - got '$value'\n";
2014     } elsif ($type eq 'string') {
2015         if (my $fmt = $confdesc->{$key}->{format}) {
2016             if ($fmt eq 'pve-qm-drive') {
2017                 # special case - we need to pass $key to parse_drive()
2018                 my $drive = parse_drive($key, $value);
2019                 return $value if $drive;
2020                 die "unable to parse drive options\n";
2021             }
2022             PVE::JSONSchema::check_format($fmt, $value);
2023             return $value;
2024         }
2025         $value =~ s/^\"(.*)\"$/$1/;
2026         return $value;
2027     } else {
2028         die "internal error"
2029     }
2030 }
2031
2032 sub check_iommu_support{
2033     #fixme : need to check IOMMU support
2034     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2035
2036     my $iommu=1;
2037     return $iommu;
2038
2039 }
2040
2041 sub touch_config {
2042     my ($vmid) = @_;
2043
2044     my $conf = PVE::QemuConfig->config_file($vmid);
2045     utime undef, undef, $conf;
2046 }
2047
2048 sub destroy_vm {
2049     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2050
2051     my $conffile = PVE::QemuConfig->config_file($vmid);
2052
2053     my $conf = PVE::QemuConfig->load_config($vmid);
2054
2055     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2056
2057     # only remove disks owned by this VM
2058     foreach_drive($conf, sub {
2059         my ($ds, $drive) = @_;
2060
2061         return if drive_is_cdrom($drive);
2062
2063         my $volid = $drive->{file};
2064
2065         return if !$volid || $volid =~ m|^/|;
2066
2067         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2068         return if !$path || !$owner || ($owner != $vmid);
2069
2070         PVE::Storage::vdisk_free($storecfg, $volid);
2071     });
2072
2073     if ($keep_empty_config) {
2074         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2075     } else {
2076         unlink $conffile;
2077     }
2078
2079     # also remove unused disk
2080     eval {
2081         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2082
2083         eval {
2084             PVE::Storage::foreach_volid($dl, sub {
2085                 my ($volid, $sid, $volname, $d) = @_;
2086                 PVE::Storage::vdisk_free($storecfg, $volid);
2087             });
2088         };
2089         warn $@ if $@;
2090
2091     };
2092     warn $@ if $@;
2093 }
2094
2095 sub parse_vm_config {
2096     my ($filename, $raw) = @_;
2097
2098     return undef if !defined($raw);
2099
2100     my $res = {
2101         digest => Digest::SHA::sha1_hex($raw),
2102         snapshots => {},
2103         pending => {},
2104     };
2105
2106     $filename =~ m|/qemu-server/(\d+)\.conf$|
2107         || die "got strange filename '$filename'";
2108
2109     my $vmid = $1;
2110
2111     my $conf = $res;
2112     my $descr;
2113     my $section = '';
2114
2115     my @lines = split(/\n/, $raw);
2116     foreach my $line (@lines) {
2117         next if $line =~ m/^\s*$/;
2118
2119         if ($line =~ m/^\[PENDING\]\s*$/i) {
2120             $section = 'pending';
2121             if (defined($descr)) {
2122                 $descr =~ s/\s+$//;
2123                 $conf->{description} = $descr;
2124             }
2125             $descr = undef;
2126             $conf = $res->{$section} = {};
2127             next;
2128
2129         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2130             $section = $1;
2131             if (defined($descr)) {
2132                 $descr =~ s/\s+$//;
2133                 $conf->{description} = $descr;
2134             }
2135             $descr = undef;
2136             $conf = $res->{snapshots}->{$section} = {};
2137             next;
2138         }
2139
2140         if ($line =~ m/^\#(.*)\s*$/) {
2141             $descr = '' if !defined($descr);
2142             $descr .= PVE::Tools::decode_text($1) . "\n";
2143             next;
2144         }
2145
2146         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2147             $descr = '' if !defined($descr);
2148             $descr .= PVE::Tools::decode_text($2);
2149         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2150             $conf->{snapstate} = $1;
2151         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2152             my $key = $1;
2153             my $value = $2;
2154             $conf->{$key} = $value;
2155         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2156             my $value = $1;
2157             if ($section eq 'pending') {
2158                 $conf->{delete} = $value; # we parse this later
2159             } else {
2160                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2161             }
2162         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2163             my $key = $1;
2164             my $value = $2;
2165             eval { $value = check_type($key, $value); };
2166             if ($@) {
2167                 warn "vm $vmid - unable to parse value of '$key' - $@";
2168             } else {
2169                 my $fmt = $confdesc->{$key}->{format};
2170                 if ($fmt && $fmt eq 'pve-qm-drive') {
2171                     my $v = parse_drive($key, $value);
2172                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2173                         $v->{file} = $volid;
2174                         $value = print_drive($vmid, $v);
2175                     } else {
2176                         warn "vm $vmid - unable to parse value of '$key'\n";
2177                         next;
2178                     }
2179                 }
2180
2181                 if ($key eq 'cdrom') {
2182                     $conf->{ide2} = $value;
2183                 } else {
2184                     $conf->{$key} = $value;
2185                 }
2186             }
2187         }
2188     }
2189
2190     if (defined($descr)) {
2191         $descr =~ s/\s+$//;
2192         $conf->{description} = $descr;
2193     }
2194     delete $res->{snapstate}; # just to be sure
2195
2196     return $res;
2197 }
2198
2199 sub write_vm_config {
2200     my ($filename, $conf) = @_;
2201
2202     delete $conf->{snapstate}; # just to be sure
2203
2204     if ($conf->{cdrom}) {
2205         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2206         $conf->{ide2} = $conf->{cdrom};
2207         delete $conf->{cdrom};
2208     }
2209
2210     # we do not use 'smp' any longer
2211     if ($conf->{sockets}) {
2212         delete $conf->{smp};
2213     } elsif ($conf->{smp}) {
2214         $conf->{sockets} = $conf->{smp};
2215         delete $conf->{cores};
2216         delete $conf->{smp};
2217     }
2218
2219     my $used_volids = {};
2220
2221     my $cleanup_config = sub {
2222         my ($cref, $pending, $snapname) = @_;
2223
2224         foreach my $key (keys %$cref) {
2225             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2226                 $key eq 'snapstate' || $key eq 'pending';
2227             my $value = $cref->{$key};
2228             if ($key eq 'delete') {
2229                 die "propertry 'delete' is only allowed in [PENDING]\n"
2230                     if !$pending;
2231                 # fixme: check syntax?
2232                 next;
2233             }
2234             eval { $value = check_type($key, $value); };
2235             die "unable to parse value of '$key' - $@" if $@;
2236
2237             $cref->{$key} = $value;
2238
2239             if (!$snapname && is_valid_drivename($key)) {
2240                 my $drive = parse_drive($key, $value);
2241                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2242             }
2243         }
2244     };
2245
2246     &$cleanup_config($conf);
2247
2248     &$cleanup_config($conf->{pending}, 1);
2249
2250     foreach my $snapname (keys %{$conf->{snapshots}}) {
2251         die "internal error" if $snapname eq 'pending';
2252         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2253     }
2254
2255     # remove 'unusedX' settings if we re-add a volume
2256     foreach my $key (keys %$conf) {
2257         my $value = $conf->{$key};
2258         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2259             delete $conf->{$key};
2260         }
2261     }
2262
2263     my $generate_raw_config = sub {
2264         my ($conf, $pending) = @_;
2265
2266         my $raw = '';
2267
2268         # add description as comment to top of file
2269         if (defined(my $descr = $conf->{description})) {
2270             if ($descr) {
2271                 foreach my $cl (split(/\n/, $descr)) {
2272                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2273                 }
2274             } else {
2275                 $raw .= "#\n" if $pending;
2276             }
2277         }
2278
2279         foreach my $key (sort keys %$conf) {
2280             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2281             $raw .= "$key: $conf->{$key}\n";
2282         }
2283         return $raw;
2284     };
2285
2286     my $raw = &$generate_raw_config($conf);
2287
2288     if (scalar(keys %{$conf->{pending}})){
2289         $raw .= "\n[PENDING]\n";
2290         $raw .= &$generate_raw_config($conf->{pending}, 1);
2291     }
2292
2293     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2294         $raw .= "\n[$snapname]\n";
2295         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2296     }
2297
2298     return $raw;
2299 }
2300
2301 sub load_defaults {
2302
2303     my $res = {};
2304
2305     # we use static defaults from our JSON schema configuration
2306     foreach my $key (keys %$confdesc) {
2307         if (defined(my $default = $confdesc->{$key}->{default})) {
2308             $res->{$key} = $default;
2309         }
2310     }
2311
2312     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2313     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2314
2315     return $res;
2316 }
2317
2318 sub config_list {
2319     my $vmlist = PVE::Cluster::get_vmlist();
2320     my $res = {};
2321     return $res if !$vmlist || !$vmlist->{ids};
2322     my $ids = $vmlist->{ids};
2323
2324     foreach my $vmid (keys %$ids) {
2325         my $d = $ids->{$vmid};
2326         next if !$d->{node} || $d->{node} ne $nodename;
2327         next if !$d->{type} || $d->{type} ne 'qemu';
2328         $res->{$vmid}->{exists} = 1;
2329     }
2330     return $res;
2331 }
2332
2333 # test if VM uses local resources (to prevent migration)
2334 sub check_local_resources {
2335     my ($conf, $noerr) = @_;
2336
2337     my $loc_res = 0;
2338
2339     $loc_res = 1 if $conf->{hostusb}; # old syntax
2340     $loc_res = 1 if $conf->{hostpci}; # old syntax
2341
2342     foreach my $k (keys %$conf) {
2343         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2344         # sockets are safe: they will recreated be on the target side post-migrate
2345         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2346         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2347     }
2348
2349     die "VM uses local resources\n" if $loc_res && !$noerr;
2350
2351     return $loc_res;
2352 }
2353
2354 # check if used storages are available on all nodes (use by migrate)
2355 sub check_storage_availability {
2356     my ($storecfg, $conf, $node) = @_;
2357
2358     foreach_drive($conf, sub {
2359         my ($ds, $drive) = @_;
2360
2361         my $volid = $drive->{file};
2362         return if !$volid;
2363
2364         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2365         return if !$sid;
2366
2367         # check if storage is available on both nodes
2368         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2369         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2370    });
2371 }
2372
2373 # list nodes where all VM images are available (used by has_feature API)
2374 sub shared_nodes {
2375     my ($conf, $storecfg) = @_;
2376
2377     my $nodelist = PVE::Cluster::get_nodelist();
2378     my $nodehash = { map { $_ => 1 } @$nodelist };
2379     my $nodename = PVE::INotify::nodename();
2380
2381     foreach_drive($conf, sub {
2382         my ($ds, $drive) = @_;
2383
2384         my $volid = $drive->{file};
2385         return if !$volid;
2386
2387         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2388         if ($storeid) {
2389             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2390             if ($scfg->{disable}) {
2391                 $nodehash = {};
2392             } elsif (my $avail = $scfg->{nodes}) {
2393                 foreach my $node (keys %$nodehash) {
2394                     delete $nodehash->{$node} if !$avail->{$node};
2395                 }
2396             } elsif (!$scfg->{shared}) {
2397                 foreach my $node (keys %$nodehash) {
2398                     delete $nodehash->{$node} if $node ne $nodename
2399                 }
2400             }
2401         }
2402     });
2403
2404     return $nodehash
2405 }
2406
2407 sub check_cmdline {
2408     my ($pidfile, $pid) = @_;
2409
2410     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2411     if (defined($fh)) {
2412         my $line = <$fh>;
2413         $fh->close;
2414         return undef if !$line;
2415         my @param = split(/\0/, $line);
2416
2417         my $cmd = $param[0];
2418         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2419
2420         for (my $i = 0; $i < scalar (@param); $i++) {
2421             my $p = $param[$i];
2422             next if !$p;
2423             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2424                 my $p = $param[$i+1];
2425                 return 1 if $p && ($p eq $pidfile);
2426                 return undef;
2427             }
2428         }
2429     }
2430     return undef;
2431 }
2432
2433 sub check_running {
2434     my ($vmid, $nocheck, $node) = @_;
2435
2436     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2437
2438     die "unable to find configuration file for VM $vmid - no such machine\n"
2439         if !$nocheck && ! -f $filename;
2440
2441     my $pidfile = pidfile_name($vmid);
2442
2443     if (my $fd = IO::File->new("<$pidfile")) {
2444         my $st = stat($fd);
2445         my $line = <$fd>;
2446         close($fd);
2447
2448         my $mtime = $st->mtime;
2449         if ($mtime > time()) {
2450             warn "file '$filename' modified in future\n";
2451         }
2452
2453         if ($line =~ m/^(\d+)$/) {
2454             my $pid = $1;
2455             if (check_cmdline($pidfile, $pid)) {
2456                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2457                     return $pid;
2458                 }
2459             }
2460         }
2461     }
2462
2463     return undef;
2464 }
2465
2466 sub vzlist {
2467
2468     my $vzlist = config_list();
2469
2470     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2471
2472     while (defined(my $de = $fd->read)) {
2473         next if $de !~ m/^(\d+)\.pid$/;
2474         my $vmid = $1;
2475         next if !defined($vzlist->{$vmid});
2476         if (my $pid = check_running($vmid)) {
2477             $vzlist->{$vmid}->{pid} = $pid;
2478         }
2479     }
2480
2481     return $vzlist;
2482 }
2483
2484 sub disksize {
2485     my ($storecfg, $conf) = @_;
2486
2487     my $bootdisk = $conf->{bootdisk};
2488     return undef if !$bootdisk;
2489     return undef if !is_valid_drivename($bootdisk);
2490
2491     return undef if !$conf->{$bootdisk};
2492
2493     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2494     return undef if !defined($drive);
2495
2496     return undef if drive_is_cdrom($drive);
2497
2498     my $volid = $drive->{file};
2499     return undef if !$volid;
2500
2501     return $drive->{size};
2502 }
2503
2504 my $last_proc_pid_stat;
2505
2506 # get VM status information
2507 # This must be fast and should not block ($full == false)
2508 # We only query KVM using QMP if $full == true (this can be slow)
2509 sub vmstatus {
2510     my ($opt_vmid, $full) = @_;
2511
2512     my $res = {};
2513
2514     my $storecfg = PVE::Storage::config();
2515
2516     my $list = vzlist();
2517     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2518
2519     my $cpucount = $cpuinfo->{cpus} || 1;
2520
2521     foreach my $vmid (keys %$list) {
2522         next if $opt_vmid && ($vmid ne $opt_vmid);
2523
2524         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2525         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2526
2527         my $d = {};
2528         $d->{pid} = $list->{$vmid}->{pid};
2529
2530         # fixme: better status?
2531         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2532
2533         my $size = disksize($storecfg, $conf);
2534         if (defined($size)) {
2535             $d->{disk} = 0; # no info available
2536             $d->{maxdisk} = $size;
2537         } else {
2538             $d->{disk} = 0;
2539             $d->{maxdisk} = 0;
2540         }
2541
2542         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2543         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2544         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2545
2546         $d->{name} = $conf->{name} || "VM $vmid";
2547         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2548
2549         if ($conf->{balloon}) {
2550             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2551             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2552         }
2553
2554         $d->{uptime} = 0;
2555         $d->{cpu} = 0;
2556         $d->{mem} = 0;
2557
2558         $d->{netout} = 0;
2559         $d->{netin} = 0;
2560
2561         $d->{diskread} = 0;
2562         $d->{diskwrite} = 0;
2563
2564         $d->{template} = PVE::QemuConfig->is_template($conf);
2565
2566         $res->{$vmid} = $d;
2567     }
2568
2569     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2570     foreach my $dev (keys %$netdev) {
2571         next if $dev !~ m/^tap([1-9]\d*)i/;
2572         my $vmid = $1;
2573         my $d = $res->{$vmid};
2574         next if !$d;
2575
2576         $d->{netout} += $netdev->{$dev}->{receive};
2577         $d->{netin} += $netdev->{$dev}->{transmit};
2578
2579         if ($full) {
2580             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2581             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2582         }
2583
2584     }
2585
2586     my $ctime = gettimeofday;
2587
2588     foreach my $vmid (keys %$list) {
2589
2590         my $d = $res->{$vmid};
2591         my $pid = $d->{pid};
2592         next if !$pid;
2593
2594         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2595         next if !$pstat; # not running
2596
2597         my $used = $pstat->{utime} + $pstat->{stime};
2598
2599         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2600
2601         if ($pstat->{vsize}) {
2602             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2603         }
2604
2605         my $old = $last_proc_pid_stat->{$pid};
2606         if (!$old) {
2607             $last_proc_pid_stat->{$pid} = {
2608                 time => $ctime,
2609                 used => $used,
2610                 cpu => 0,
2611             };
2612             next;
2613         }
2614
2615         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2616
2617         if ($dtime > 1000) {
2618             my $dutime = $used -  $old->{used};
2619
2620             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2621             $last_proc_pid_stat->{$pid} = {
2622                 time => $ctime,
2623                 used => $used,
2624                 cpu => $d->{cpu},
2625             };
2626         } else {
2627             $d->{cpu} = $old->{cpu};
2628         }
2629     }
2630
2631     return $res if !$full;
2632
2633     my $qmpclient = PVE::QMPClient->new();
2634
2635     my $ballooncb = sub {
2636         my ($vmid, $resp) = @_;
2637
2638         my $info = $resp->{'return'};
2639         return if !$info->{max_mem};
2640
2641         my $d = $res->{$vmid};
2642
2643         # use memory assigned to VM
2644         $d->{maxmem} = $info->{max_mem};
2645         $d->{balloon} = $info->{actual};
2646
2647         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2648             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2649             $d->{freemem} = $info->{free_mem};
2650         }
2651
2652         $d->{ballooninfo} = $info;
2653     };
2654
2655     my $blockstatscb = sub {
2656         my ($vmid, $resp) = @_;
2657         my $data = $resp->{'return'} || [];
2658         my $totalrdbytes = 0;
2659         my $totalwrbytes = 0;
2660
2661         for my $blockstat (@$data) {
2662             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2663             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2664
2665             $blockstat->{device} =~ s/drive-//;
2666             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2667         }
2668         $res->{$vmid}->{diskread} = $totalrdbytes;
2669         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2670     };
2671
2672     my $statuscb = sub {
2673         my ($vmid, $resp) = @_;
2674
2675         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2676         # this fails if ballon driver is not loaded, so this must be
2677         # the last commnand (following command are aborted if this fails).
2678         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2679
2680         my $status = 'unknown';
2681         if (!defined($status = $resp->{'return'}->{status})) {
2682             warn "unable to get VM status\n";
2683             return;
2684         }
2685
2686         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2687     };
2688
2689     foreach my $vmid (keys %$list) {
2690         next if $opt_vmid && ($vmid ne $opt_vmid);
2691         next if !$res->{$vmid}->{pid}; # not running
2692         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2693     }
2694
2695     $qmpclient->queue_execute(undef, 1);
2696
2697     foreach my $vmid (keys %$list) {
2698         next if $opt_vmid && ($vmid ne $opt_vmid);
2699         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2700     }
2701
2702     return $res;
2703 }
2704
2705 sub foreach_drive {
2706     my ($conf, $func, @param) = @_;
2707
2708     foreach my $ds (valid_drive_names()) {
2709         next if !defined($conf->{$ds});
2710
2711         my $drive = parse_drive($ds, $conf->{$ds});
2712         next if !$drive;
2713
2714         &$func($ds, $drive, @param);
2715     }
2716 }
2717
2718 sub foreach_volid {
2719     my ($conf, $func, @param) = @_;
2720
2721     my $volhash = {};
2722
2723     my $test_volid = sub {
2724         my ($volid, $is_cdrom) = @_;
2725
2726         return if !$volid;
2727
2728         $volhash->{$volid} = $is_cdrom || 0;
2729     };
2730
2731     foreach_drive($conf, sub {
2732         my ($ds, $drive) = @_;
2733         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2734     });
2735
2736     foreach my $snapname (keys %{$conf->{snapshots}}) {
2737         my $snap = $conf->{snapshots}->{$snapname};
2738         &$test_volid($snap->{vmstate}, 0);
2739         foreach_drive($snap, sub {
2740             my ($ds, $drive) = @_;
2741             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2742         });
2743     }
2744
2745     foreach my $volid (keys %$volhash) {
2746         &$func($volid, $volhash->{$volid}, @param);
2747     }
2748 }
2749
2750 sub vga_conf_has_spice {
2751     my ($vga) = @_;
2752
2753     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2754
2755     return $1 || 1;
2756 }
2757
2758 sub config_to_command {
2759     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2760
2761     my $cmd = [];
2762     my $globalFlags = [];
2763     my $machineFlags = [];
2764     my $rtcFlags = [];
2765     my $cpuFlags = [];
2766     my $devices = [];
2767     my $pciaddr = '';
2768     my $bridges = {};
2769     my $kvmver = kvm_user_version();
2770     my $vernum = 0; # unknown
2771     my $ostype = $conf->{ostype};
2772     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2773         $vernum = $1*1000000+$2*1000;
2774     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2775         $vernum = $1*1000000+$2*1000+$3;
2776     }
2777
2778     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2779
2780     my $have_ovz = -f '/proc/vz/vestat';
2781
2782     my $q35 = machine_type_is_q35($conf);
2783     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2784     my $machine_type = $forcemachine || $conf->{machine};
2785     my $use_old_bios_files = undef;
2786     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2787
2788     my $cpuunits = defined($conf->{cpuunits}) ?
2789             $conf->{cpuunits} : $defaults->{cpuunits};
2790
2791     push @$cmd, '/usr/bin/kvm';
2792
2793     push @$cmd, '-id', $vmid;
2794
2795     my $use_virtio = 0;
2796
2797     my $qmpsocket = qmp_socket($vmid);
2798     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2799     push @$cmd, '-mon', "chardev=qmp,mode=control";
2800
2801
2802     push @$cmd, '-pidfile' , pidfile_name($vmid);
2803
2804     push @$cmd, '-daemonize';
2805
2806     if ($conf->{smbios1}) {
2807         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2808     }
2809
2810     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2811         my $ovmfbase;
2812
2813         # prefer the OVMF_CODE variant
2814         if (-f $OVMF_CODE) {
2815             $ovmfbase = $OVMF_CODE;
2816         } elsif (-f $OVMF_IMG) {
2817             $ovmfbase = $OVMF_IMG;
2818         }
2819
2820         die "no uefi base img found\n" if !$ovmfbase;
2821         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2822
2823         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2824             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2825             my $format = $d->{format} // 'raw';
2826             my $path;
2827             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2828             if ($storeid) {
2829                 $path = PVE::Storage::path($storecfg, $d->{file});
2830                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2831                 $format = qemu_img_format($scfg, $volname);
2832             } else {
2833                 $path = $d->{file};
2834                 $format = "raw";
2835             }
2836             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2837         } elsif ($ovmfbase eq $OVMF_CODE) {
2838             warn "using uefi without permanent efivars disk\n";
2839             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2840             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2841             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2842         } else {
2843             # if the base img is not OVMF_CODE, we do not have to bother
2844             # to create/use a vars image, since it will not be used anyway
2845             # this can only happen if someone manually deletes the OVMF_CODE image
2846             # or has an old pve-qemu-kvm version installed.
2847             # both should not happen, but we ignore it here
2848         }
2849     }
2850
2851
2852     # add usb controllers
2853     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2854     push @$devices, @usbcontrollers if @usbcontrollers;
2855     my $vga = $conf->{vga};
2856
2857     my $qxlnum = vga_conf_has_spice($vga);
2858     $vga = 'qxl' if $qxlnum;
2859
2860     if (!$vga) {
2861         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2862                                 $conf->{ostype} eq 'win7' ||
2863                                 $conf->{ostype} eq 'w2k8')) {
2864             $vga = 'std';
2865         } else {
2866             $vga = 'cirrus';
2867         }
2868     }
2869
2870     # enable absolute mouse coordinates (needed by vnc)
2871     my $tablet;
2872     if (defined($conf->{tablet})) {
2873         $tablet = $conf->{tablet};
2874     } else {
2875         $tablet = $defaults->{tablet};
2876         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2877         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2878     }
2879
2880     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2881
2882     my $kvm_off = 0;
2883     # host pci devices
2884     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2885         my $d = parse_hostpci($conf->{"hostpci$i"});
2886         next if !$d;
2887
2888         my $pcie = $d->{pcie};
2889         if($pcie){
2890             die "q35 machine model is not enabled" if !$q35;
2891             $pciaddr = print_pcie_addr("hostpci$i");
2892         }else{
2893             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2894         }
2895
2896         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2897         my $xvga = '';
2898         if ($d->{'x-vga'}) {
2899             $xvga = ',x-vga=on';
2900             $kvm_off = 1;
2901             $vga = 'none';
2902             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2903                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2904             }
2905             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2906                 $xvga = "";
2907             }
2908         }
2909         my $pcidevices = $d->{pciid};
2910         my $multifunction = 1 if @$pcidevices > 1;
2911
2912         my $j=0;
2913         foreach my $pcidevice (@$pcidevices) {
2914
2915             my $id = "hostpci$i";
2916             $id .= ".$j" if $multifunction;
2917             my $addr = $pciaddr;
2918             $addr .= ".$j" if $multifunction;
2919             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2920
2921             if($j == 0){
2922                 $devicestr .= "$rombar$xvga";
2923                 $devicestr .= ",multifunction=on" if $multifunction;
2924             }
2925
2926             push @$devices, '-device', $devicestr;
2927             $j++;
2928         }
2929     }
2930
2931     # usb devices
2932     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2933     push @$devices, @usbdevices if @usbdevices;
2934     # serial devices
2935     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2936         if (my $path = $conf->{"serial$i"}) {
2937             if ($path eq 'socket') {
2938                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2939                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2940                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2941             } else {
2942                 die "no such serial device\n" if ! -c $path;
2943                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2944                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2945             }
2946         }
2947     }
2948
2949     # parallel devices
2950     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2951         if (my $path = $conf->{"parallel$i"}) {
2952             die "no such parallel device\n" if ! -c $path;
2953             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2954             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2955             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2956         }
2957     }
2958
2959     my $vmname = $conf->{name} || "vm$vmid";
2960
2961     push @$cmd, '-name', $vmname;
2962
2963     my $sockets = 1;
2964     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2965     $sockets = $conf->{sockets} if  $conf->{sockets};
2966
2967     my $cores = $conf->{cores} || 1;
2968
2969     my $maxcpus = $sockets * $cores;
2970
2971     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2972
2973     my $allowed_vcpus = $cpuinfo->{cpus};
2974
2975     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2976         if ($allowed_vcpus < $maxcpus);
2977
2978     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
2979
2980         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2981         for (my $i = 2; $i <= $vcpus; $i++)  {
2982             my $cpustr = print_cpu_device($conf,$i);
2983             push @$cmd, '-device', $cpustr;
2984         }
2985
2986     } else {
2987
2988         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2989     }
2990     push @$cmd, '-nodefaults';
2991
2992     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2993
2994     my $bootindex_hash = {};
2995     my $i = 1;
2996     foreach my $o (split(//, $bootorder)) {
2997         $bootindex_hash->{$o} = $i*100;
2998         $i++;
2999     }
3000
3001     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3002
3003     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3004
3005     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3006
3007     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3008
3009     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3010         my $socket = vnc_socket($vmid);
3011         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3012     } else {
3013         push @$cmd, '-nographic';
3014     }
3015
3016     # time drift fix
3017     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3018
3019     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3020     my $useLocaltime = $conf->{localtime};
3021
3022     if ($ostype) {
3023         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3024
3025         if ($ostype =~ m/^w/) { # windows
3026             $useLocaltime = 1 if !defined($conf->{localtime});
3027
3028             # use time drift fix when acpi is enabled
3029             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3030                 $tdf = 1 if !defined($conf->{tdf});
3031             }
3032         }
3033
3034         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3035             $ostype eq 'wvista') {
3036             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3037             push @$cmd, '-no-hpet';
3038             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3039                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3040                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3041                 push @$cpuFlags , 'hv_time' if !$nokvm;
3042
3043                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
3044                     push @$cpuFlags , 'hv_reset' if !$nokvm;
3045                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
3046                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
3047                 }
3048
3049             } else {
3050                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3051             }
3052         }
3053
3054         if ($ostype eq 'win7' || $ostype eq 'win8') {
3055             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3056         }
3057     }
3058
3059     push @$rtcFlags, 'driftfix=slew' if $tdf;
3060
3061     if ($nokvm) {
3062         push @$machineFlags, 'accel=tcg';
3063     } else {
3064         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3065     }
3066
3067     if ($machine_type) {
3068         push @$machineFlags, "type=${machine_type}";
3069     }
3070
3071     if ($conf->{startdate}) {
3072         push @$rtcFlags, "base=$conf->{startdate}";
3073     } elsif ($useLocaltime) {
3074         push @$rtcFlags, 'base=localtime';
3075     }
3076
3077     my $cpu = $nokvm ? "qemu64" : "kvm64";
3078     if (my $cputype = $conf->{cpu}) {
3079         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3080             or die "Cannot parse cpu description: $cputype\n";
3081         $cpu = $cpuconf->{cputype};
3082         $kvm_off = 1 if $cpuconf->{hidden};
3083     }
3084
3085     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3086
3087     push @$cpuFlags , '-x2apic'
3088         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3089
3090     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3091
3092     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3093
3094     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3095
3096         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3097         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3098     }
3099
3100     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3101
3102     push @$cpuFlags, 'kvm=off' if $kvm_off;
3103
3104     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3105         die "internal error"; # should not happen
3106
3107     push @$cpuFlags, "vendor=${cpu_vendor}"
3108         if $cpu_vendor ne 'default';
3109
3110     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3111
3112     push @$cmd, '-cpu', $cpu;
3113
3114     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3115
3116     push @$cmd, '-S' if $conf->{freeze};
3117
3118     # set keyboard layout
3119     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3120     push @$cmd, '-k', $kb if $kb;
3121
3122     # enable sound
3123     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3124     #push @$cmd, '-soundhw', 'es1370';
3125     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3126
3127     if($conf->{agent}) {
3128         my $qgasocket = qmp_socket($vmid, 1);
3129         my $pciaddr = print_pci_addr("qga0", $bridges);
3130         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3131         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3132         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3133     }
3134
3135     my $spice_port;
3136
3137     if ($qxlnum) {
3138         if ($qxlnum > 1) {
3139             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3140                 for(my $i = 1; $i < $qxlnum; $i++){
3141                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3142                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3143                 }
3144             } else {
3145                 # assume other OS works like Linux
3146                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3147                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3148             }
3149         }
3150
3151         my $pciaddr = print_pci_addr("spice", $bridges);
3152
3153         my $nodename = PVE::INotify::nodename();
3154         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3155         $spice_port = PVE::Tools::next_spice_port($pfamily);
3156
3157         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3158
3159         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3160         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3161         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3162     }
3163
3164     # enable balloon by default, unless explicitly disabled
3165     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3166         $pciaddr = print_pci_addr("balloon0", $bridges);
3167         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3168     }
3169
3170     if ($conf->{watchdog}) {
3171         my $wdopts = parse_watchdog($conf->{watchdog});
3172         $pciaddr = print_pci_addr("watchdog", $bridges);
3173         my $watchdog = $wdopts->{model} || 'i6300esb';
3174         push @$devices, '-device', "$watchdog$pciaddr";
3175         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3176     }
3177
3178     my $vollist = [];
3179     my $scsicontroller = {};
3180     my $ahcicontroller = {};
3181     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3182
3183     # Add iscsi initiator name if available
3184     if (my $initiator = get_initiator_name()) {
3185         push @$devices, '-iscsi', "initiator-name=$initiator";
3186     }
3187
3188     foreach_drive($conf, sub {
3189         my ($ds, $drive) = @_;
3190
3191         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3192             push @$vollist, $drive->{file};
3193         }
3194
3195         $use_virtio = 1 if $ds =~ m/^virtio/;
3196
3197         if (drive_is_cdrom ($drive)) {
3198             if ($bootindex_hash->{d}) {
3199                 $drive->{bootindex} = $bootindex_hash->{d};
3200                 $bootindex_hash->{d} += 1;
3201             }
3202         } else {
3203             if ($bootindex_hash->{c}) {
3204                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3205                 $bootindex_hash->{c} += 1;
3206             }
3207         }
3208
3209         if($drive->{interface} eq 'virtio'){
3210            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3211         }
3212
3213         if ($drive->{interface} eq 'scsi') {
3214
3215             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3216
3217             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3218             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3219
3220             my $iothread = '';
3221             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3222                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3223                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3224             } elsif ($drive->{iothread}) {
3225                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3226             }
3227
3228             my $queues = '';
3229             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3230                 $queues = ",num_queues=$drive->{queues}";
3231             }
3232
3233             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3234             $scsicontroller->{$controller}=1;
3235         }
3236
3237         if ($drive->{interface} eq 'sata') {
3238            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3239            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3240            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3241            $ahcicontroller->{$controller}=1;
3242         }
3243
3244         if ($drive->{interface} eq 'efidisk') {
3245             # this will be added somewhere else
3246             return;
3247         }
3248
3249         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3250         push @$devices, '-drive',$drive_cmd;
3251         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3252     });
3253
3254     for (my $i = 0; $i < $MAX_NETS; $i++) {
3255          next if !$conf->{"net$i"};
3256          my $d = parse_net($conf->{"net$i"});
3257          next if !$d;
3258
3259          $use_virtio = 1 if $d->{model} eq 'virtio';
3260
3261          if ($bootindex_hash->{n}) {
3262             $d->{bootindex} = $bootindex_hash->{n};
3263             $bootindex_hash->{n} += 1;
3264          }
3265
3266          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3267          push @$devices, '-netdev', $netdevfull;
3268
3269          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3270          push @$devices, '-device', $netdevicefull;
3271     }
3272
3273     if (!$q35) {
3274         # add pci bridges
3275         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3276            $bridges->{1} = 1;
3277            $bridges->{2} = 1;
3278         }
3279
3280         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3281
3282         while (my ($k, $v) = each %$bridges) {
3283             $pciaddr = print_pci_addr("pci.$k");
3284             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3285         }
3286     }
3287
3288     # add custom args
3289     if ($conf->{args}) {
3290         my $aa = PVE::Tools::split_args($conf->{args});
3291         push @$cmd, @$aa;
3292     }
3293
3294     push @$cmd, @$devices;
3295     push @$cmd, '-rtc', join(',', @$rtcFlags)
3296         if scalar(@$rtcFlags);
3297     push @$cmd, '-machine', join(',', @$machineFlags)
3298         if scalar(@$machineFlags);
3299     push @$cmd, '-global', join(',', @$globalFlags)
3300         if scalar(@$globalFlags);
3301
3302     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3303 }
3304
3305 sub vnc_socket {
3306     my ($vmid) = @_;
3307     return "${var_run_tmpdir}/$vmid.vnc";
3308 }
3309
3310 sub spice_port {
3311     my ($vmid) = @_;
3312
3313     my $res = vm_mon_cmd($vmid, 'query-spice');
3314
3315     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3316 }
3317
3318 sub qmp_socket {
3319     my ($vmid, $qga) = @_;
3320     my $sockettype = $qga ? 'qga' : 'qmp';
3321     return "${var_run_tmpdir}/$vmid.$sockettype";
3322 }
3323
3324 sub pidfile_name {
3325     my ($vmid) = @_;
3326     return "${var_run_tmpdir}/$vmid.pid";
3327 }
3328
3329 sub vm_devices_list {
3330     my ($vmid) = @_;
3331
3332     my $res = vm_mon_cmd($vmid, 'query-pci');
3333     my $devices = {};
3334     foreach my $pcibus (@$res) {
3335         foreach my $device (@{$pcibus->{devices}}) {
3336             next if !$device->{'qdev_id'};
3337             if ($device->{'pci_bridge'}) {
3338                 $devices->{$device->{'qdev_id'}} = 1;
3339                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3340                     next if !$bridge_device->{'qdev_id'};
3341                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3342                     $devices->{$device->{'qdev_id'}}++;
3343                 }
3344             } else {
3345                 $devices->{$device->{'qdev_id'}} = 1;
3346             }
3347         }
3348     }
3349
3350     my $resblock = vm_mon_cmd($vmid, 'query-block');
3351     foreach my $block (@$resblock) {
3352         if($block->{device} =~ m/^drive-(\S+)/){
3353                 $devices->{$1} = 1;
3354         }
3355     }
3356
3357     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3358     foreach my $mice (@$resmice) {
3359         if ($mice->{name} eq 'QEMU HID Tablet') {
3360             $devices->{tablet} = 1;
3361             last;
3362         }
3363     }
3364
3365     # for usb devices there is no query-usb
3366     # but we can iterate over the entries in
3367     # qom-list path=/machine/peripheral
3368     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3369     foreach my $per (@$resperipheral) {
3370         if ($per->{name} =~ m/^usb\d+$/) {
3371             $devices->{$per->{name}} = 1;
3372         }
3373     }
3374
3375     return $devices;
3376 }
3377
3378 sub vm_deviceplug {
3379     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3380
3381     my $q35 = machine_type_is_q35($conf);
3382
3383     my $devices_list = vm_devices_list($vmid);
3384     return 1 if defined($devices_list->{$deviceid});
3385
3386     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3387
3388     if ($deviceid eq 'tablet') {
3389
3390         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3391
3392     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3393
3394         die "usb hotplug currently not reliable\n";
3395         # since we can't reliably hot unplug all added usb devices
3396         # and usb passthrough disables live migration
3397         # we disable usb hotplugging for now
3398         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3399
3400     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3401
3402         qemu_iothread_add($vmid, $deviceid, $device);
3403
3404         qemu_driveadd($storecfg, $vmid, $device);
3405         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3406
3407         qemu_deviceadd($vmid, $devicefull);
3408         eval { qemu_deviceaddverify($vmid, $deviceid); };
3409         if (my $err = $@) {
3410             eval { qemu_drivedel($vmid, $deviceid); };
3411             warn $@ if $@;
3412             die $err;
3413         }
3414
3415     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3416
3417
3418         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3419         my $pciaddr = print_pci_addr($deviceid);
3420         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3421
3422         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3423
3424         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3425             qemu_iothread_add($vmid, $deviceid, $device);
3426             $devicefull .= ",iothread=iothread-$deviceid";
3427         }
3428
3429         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3430             $devicefull .= ",num_queues=$device->{queues}";
3431         }
3432
3433         qemu_deviceadd($vmid, $devicefull);
3434         qemu_deviceaddverify($vmid, $deviceid);
3435
3436     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3437
3438         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3439         qemu_driveadd($storecfg, $vmid, $device);
3440
3441         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3442         eval { qemu_deviceadd($vmid, $devicefull); };
3443         if (my $err = $@) {
3444             eval { qemu_drivedel($vmid, $deviceid); };
3445             warn $@ if $@;
3446             die $err;
3447         }
3448
3449     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3450
3451         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3452
3453         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3454         my $use_old_bios_files = undef;
3455         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3456
3457         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3458         qemu_deviceadd($vmid, $netdevicefull);
3459         eval { qemu_deviceaddverify($vmid, $deviceid); };
3460         if (my $err = $@) {
3461             eval { qemu_netdevdel($vmid, $deviceid); };
3462             warn $@ if $@;
3463             die $err;
3464         }
3465
3466     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3467
3468         my $bridgeid = $2;
3469         my $pciaddr = print_pci_addr($deviceid);
3470         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3471
3472         qemu_deviceadd($vmid, $devicefull);
3473         qemu_deviceaddverify($vmid, $deviceid);
3474
3475     } else {
3476         die "can't hotplug device '$deviceid'\n";
3477     }
3478
3479     return 1;
3480 }
3481
3482 # fixme: this should raise exceptions on error!
3483 sub vm_deviceunplug {
3484     my ($vmid, $conf, $deviceid) = @_;
3485
3486     my $devices_list = vm_devices_list($vmid);
3487     return 1 if !defined($devices_list->{$deviceid});
3488
3489     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3490
3491     if ($deviceid eq 'tablet') {
3492
3493         qemu_devicedel($vmid, $deviceid);
3494
3495     } elsif ($deviceid =~ m/^usb\d+$/) {
3496
3497         die "usb hotplug currently not reliable\n";
3498         # when unplugging usb devices this way,
3499         # there may be remaining usb controllers/hubs
3500         # so we disable it for now
3501         qemu_devicedel($vmid, $deviceid);
3502         qemu_devicedelverify($vmid, $deviceid);
3503
3504     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3505
3506         qemu_devicedel($vmid, $deviceid);
3507         qemu_devicedelverify($vmid, $deviceid);
3508         qemu_drivedel($vmid, $deviceid);
3509         qemu_iothread_del($conf, $vmid, $deviceid);
3510
3511     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3512
3513         qemu_devicedel($vmid, $deviceid);
3514         qemu_devicedelverify($vmid, $deviceid);
3515         qemu_iothread_del($conf, $vmid, $deviceid);
3516
3517     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3518
3519         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3520         my $device = parse_drive($deviceid, $conf->{$deviceid});
3521         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3522
3523         qemu_devicedel($vmid, $deviceid);
3524         qemu_drivedel($vmid, $deviceid);
3525         qemu_deletescsihw($conf, $vmid, $deviceid);
3526
3527     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3528
3529         qemu_devicedel($vmid, $deviceid);
3530         qemu_devicedelverify($vmid, $deviceid);
3531         qemu_netdevdel($vmid, $deviceid);
3532
3533     } else {
3534         die "can't unplug device '$deviceid'\n";
3535     }
3536
3537     return 1;
3538 }
3539
3540 sub qemu_deviceadd {
3541     my ($vmid, $devicefull) = @_;
3542
3543     $devicefull = "driver=".$devicefull;
3544     my %options =  split(/[=,]/, $devicefull);
3545
3546     vm_mon_cmd($vmid, "device_add" , %options);
3547 }
3548
3549 sub qemu_devicedel {
3550     my ($vmid, $deviceid) = @_;
3551
3552     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3553 }
3554
3555 sub qemu_iothread_add {
3556     my($vmid, $deviceid, $device) = @_;
3557
3558     if ($device->{iothread}) {
3559         my $iothreads = vm_iothreads_list($vmid);
3560         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3561     }
3562 }
3563
3564 sub qemu_iothread_del {
3565     my($conf, $vmid, $deviceid) = @_;
3566
3567     my $device = parse_drive($deviceid, $conf->{$deviceid});
3568     if ($device->{iothread}) {
3569         my $iothreads = vm_iothreads_list($vmid);
3570         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3571     }
3572 }
3573
3574 sub qemu_objectadd {
3575     my($vmid, $objectid, $qomtype) = @_;
3576
3577     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3578
3579     return 1;
3580 }
3581
3582 sub qemu_objectdel {
3583     my($vmid, $objectid) = @_;
3584
3585     vm_mon_cmd($vmid, "object-del", id => $objectid);
3586
3587     return 1;
3588 }
3589
3590 sub qemu_driveadd {
3591     my ($storecfg, $vmid, $device) = @_;
3592
3593     my $drive = print_drive_full($storecfg, $vmid, $device);
3594     $drive =~ s/\\/\\\\/g;
3595     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3596
3597     # If the command succeeds qemu prints: "OK"
3598     return 1 if $ret =~ m/OK/s;
3599
3600     die "adding drive failed: $ret\n";
3601 }
3602
3603 sub qemu_drivedel {
3604     my($vmid, $deviceid) = @_;
3605
3606     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3607     $ret =~ s/^\s+//;
3608
3609     return 1 if $ret eq "";
3610
3611     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3612     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3613
3614     die "deleting drive $deviceid failed : $ret\n";
3615 }
3616
3617 sub qemu_deviceaddverify {
3618     my ($vmid, $deviceid) = @_;
3619
3620     for (my $i = 0; $i <= 5; $i++) {
3621          my $devices_list = vm_devices_list($vmid);
3622          return 1 if defined($devices_list->{$deviceid});
3623          sleep 1;
3624     }
3625
3626     die "error on hotplug device '$deviceid'\n";
3627 }
3628
3629
3630 sub qemu_devicedelverify {
3631     my ($vmid, $deviceid) = @_;
3632
3633     # need to verify that the device is correctly removed as device_del
3634     # is async and empty return is not reliable
3635
3636     for (my $i = 0; $i <= 5; $i++) {
3637          my $devices_list = vm_devices_list($vmid);
3638          return 1 if !defined($devices_list->{$deviceid});
3639          sleep 1;
3640     }
3641
3642     die "error on hot-unplugging device '$deviceid'\n";
3643 }
3644
3645 sub qemu_findorcreatescsihw {
3646     my ($storecfg, $conf, $vmid, $device) = @_;
3647
3648     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3649
3650     my $scsihwid="$controller_prefix$controller";
3651     my $devices_list = vm_devices_list($vmid);
3652
3653     if(!defined($devices_list->{$scsihwid})) {
3654         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3655     }
3656
3657     return 1;
3658 }
3659
3660 sub qemu_deletescsihw {
3661     my ($conf, $vmid, $opt) = @_;
3662
3663     my $device = parse_drive($opt, $conf->{$opt});
3664
3665     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3666         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3667         return 1;
3668     }
3669
3670     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3671
3672     my $devices_list = vm_devices_list($vmid);
3673     foreach my $opt (keys %{$devices_list}) {
3674         if (PVE::QemuServer::is_valid_drivename($opt)) {
3675             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3676             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3677                 return 1;
3678             }
3679         }
3680     }
3681
3682     my $scsihwid="scsihw$controller";
3683
3684     vm_deviceunplug($vmid, $conf, $scsihwid);
3685
3686     return 1;
3687 }
3688
3689 sub qemu_add_pci_bridge {
3690     my ($storecfg, $conf, $vmid, $device) = @_;
3691
3692     my $bridges = {};
3693
3694     my $bridgeid;
3695
3696     print_pci_addr($device, $bridges);
3697
3698     while (my ($k, $v) = each %$bridges) {
3699         $bridgeid = $k;
3700     }
3701     return 1 if !defined($bridgeid) || $bridgeid < 1;
3702
3703     my $bridge = "pci.$bridgeid";
3704     my $devices_list = vm_devices_list($vmid);
3705
3706     if (!defined($devices_list->{$bridge})) {
3707         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3708     }
3709
3710     return 1;
3711 }
3712
3713 sub qemu_set_link_status {
3714     my ($vmid, $device, $up) = @_;
3715
3716     vm_mon_cmd($vmid, "set_link", name => $device,
3717                up => $up ? JSON::true : JSON::false);
3718 }
3719
3720 sub qemu_netdevadd {
3721     my ($vmid, $conf, $device, $deviceid) = @_;
3722
3723     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3724     my %options =  split(/[=,]/, $netdev);
3725
3726     vm_mon_cmd($vmid, "netdev_add",  %options);
3727     return 1;
3728 }
3729
3730 sub qemu_netdevdel {
3731     my ($vmid, $deviceid) = @_;
3732
3733     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3734 }
3735
3736 sub qemu_usb_hotplug {
3737     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3738
3739     return if !$device;
3740
3741     # remove the old one first
3742     vm_deviceunplug($vmid, $conf, $deviceid);
3743
3744     # check if xhci controller is necessary and available
3745     if ($device->{usb3}) {
3746
3747         my $devicelist = vm_devices_list($vmid);
3748
3749         if (!$devicelist->{xhci}) {
3750             my $pciaddr = print_pci_addr("xhci");
3751             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3752         }
3753     }
3754     my $d = parse_usb_device($device->{host});
3755     $d->{usb3} = $device->{usb3};
3756
3757     # add the new one
3758     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3759 }
3760
3761 sub qemu_cpu_hotplug {
3762     my ($vmid, $conf, $vcpus) = @_;
3763
3764     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3765
3766     my $sockets = 1;
3767     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3768     $sockets = $conf->{sockets} if  $conf->{sockets};
3769     my $cores = $conf->{cores} || 1;
3770     my $maxcpus = $sockets * $cores;
3771
3772     $vcpus = $maxcpus if !$vcpus;
3773
3774     die "you can't add more vcpus than maxcpus\n"
3775         if $vcpus > $maxcpus;
3776
3777     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3778
3779     if ($vcpus < $currentvcpus) {
3780
3781         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3782
3783             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3784                 qemu_devicedel($vmid, "cpu$i");
3785                 my $retry = 0;
3786                 my $currentrunningvcpus = undef;
3787                 while (1) {
3788                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3789                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3790                     raise_param_exc({ "cpu unplug" => "error unplug cpu$i" }) if $retry > 5;
3791                     $retry++;
3792                     sleep 1;
3793                 }
3794                 #update conf after each succesfull cpu unplug
3795                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3796                 PVE::QemuConfig->write_config($vmid, $conf);
3797             }
3798         } else {
3799             die "online cpu unplug is only possible since qemu 2.7\n"
3800         }
3801
3802         return;
3803     }
3804
3805     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3806     die "vcpus in running vm is different than configuration\n"
3807         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3808
3809     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3810
3811         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3812             my $cpustr = print_cpu_device($conf, $i);
3813             qemu_deviceadd($vmid, $cpustr);
3814
3815             my $retry = 0;
3816             my $currentrunningvcpus = undef;
3817             while (1) {
3818                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3819                 last if scalar(@{$currentrunningvcpus}) == $i;
3820                 raise_param_exc({ "cpu hotplug" => "error hotplug cpu$i" }) if $retry > 10;
3821                 sleep 1;
3822                 $retry++;
3823             }
3824             #update conf after each succesfull cpu hotplug
3825             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3826             PVE::QemuConfig->write_config($vmid, $conf);
3827         }
3828     } else {
3829
3830         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3831             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3832         }
3833     }
3834 }
3835
3836 sub qemu_block_set_io_throttle {
3837     my ($vmid, $deviceid,
3838         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3839         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3840
3841     return if !check_running($vmid) ;
3842
3843     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3844         bps => int($bps),
3845         bps_rd => int($bps_rd),
3846         bps_wr => int($bps_wr),
3847         iops => int($iops),
3848         iops_rd => int($iops_rd),
3849         iops_wr => int($iops_wr),
3850         bps_max => int($bps_max),
3851         bps_rd_max => int($bps_rd_max),
3852         bps_wr_max => int($bps_wr_max),
3853         iops_max => int($iops_max),
3854         iops_rd_max => int($iops_rd_max),
3855         iops_wr_max => int($iops_wr_max)
3856     );
3857
3858 }
3859
3860 # old code, only used to shutdown old VM after update
3861 sub __read_avail {
3862     my ($fh, $timeout) = @_;
3863
3864     my $sel = new IO::Select;
3865     $sel->add($fh);
3866
3867     my $res = '';
3868     my $buf;
3869
3870     my @ready;
3871     while (scalar (@ready = $sel->can_read($timeout))) {
3872         my $count;
3873         if ($count = $fh->sysread($buf, 8192)) {
3874             if ($buf =~ /^(.*)\(qemu\) $/s) {
3875                 $res .= $1;
3876                 last;
3877             } else {
3878                 $res .= $buf;
3879             }
3880         } else {
3881             if (!defined($count)) {
3882                 die "$!\n";
3883             }
3884             last;
3885         }
3886     }
3887
3888     die "monitor read timeout\n" if !scalar(@ready);
3889
3890     return $res;
3891 }
3892
3893 # old code, only used to shutdown old VM after update
3894 sub vm_monitor_command {
3895     my ($vmid, $cmdstr, $nocheck) = @_;
3896
3897     my $res;
3898
3899     eval {
3900         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3901
3902         my $sname = "${var_run_tmpdir}/$vmid.mon";
3903
3904         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3905             die "unable to connect to VM $vmid socket - $!\n";
3906
3907         my $timeout = 3;
3908
3909         # hack: migrate sometime blocks the monitor (when migrate_downtime
3910         # is set)
3911         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3912             $timeout = 60*60; # 1 hour
3913         }
3914
3915         # read banner;
3916         my $data = __read_avail($sock, $timeout);
3917
3918         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3919             die "got unexpected qemu monitor banner\n";
3920         }
3921
3922         my $sel = new IO::Select;
3923         $sel->add($sock);
3924
3925         if (!scalar(my @ready = $sel->can_write($timeout))) {
3926             die "monitor write error - timeout";
3927         }
3928
3929         my $fullcmd = "$cmdstr\r";
3930
3931         # syslog('info', "VM $vmid monitor command: $cmdstr");
3932
3933         my $b;
3934         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3935             die "monitor write error - $!";
3936         }
3937
3938         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3939
3940         $timeout = 20;
3941
3942         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3943             $timeout = 60*60; # 1 hour
3944         } elsif ($cmdstr =~ m/^(eject|change)/) {
3945             $timeout = 60; # note: cdrom mount command is slow
3946         }
3947         if ($res = __read_avail($sock, $timeout)) {
3948
3949             my @lines = split("\r?\n", $res);
3950
3951             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3952
3953             $res = join("\n", @lines);
3954             $res .= "\n";
3955         }
3956     };
3957
3958     my $err = $@;
3959
3960     if ($err) {
3961         syslog("err", "VM $vmid monitor command failed - $err");
3962         die $err;
3963     }
3964
3965     return $res;
3966 }
3967
3968 sub qemu_block_resize {
3969     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3970
3971     my $running = check_running($vmid);
3972
3973     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3974
3975     return if !$running;
3976
3977     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3978
3979 }
3980
3981 sub qemu_volume_snapshot {
3982     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3983
3984     my $running = check_running($vmid);
3985
3986     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3987         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3988     } else {
3989         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3990     }
3991 }
3992
3993 sub qemu_volume_snapshot_delete {
3994     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3995
3996     my $running = check_running($vmid);
3997
3998     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3999
4000     return if !$running;
4001
4002     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4003 }
4004
4005 sub set_migration_caps {
4006     my ($vmid) = @_;
4007
4008     my $cap_ref = [];
4009
4010     my $enabled_cap = {
4011         "auto-converge" => 1,
4012         "xbzrle" => 1,
4013         "x-rdma-pin-all" => 0,
4014         "zero-blocks" => 0,
4015         "compress" => 0
4016     };
4017
4018     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4019
4020     for my $supported_capability (@$supported_capabilities) {
4021         push @$cap_ref, {
4022             capability => $supported_capability->{capability},
4023             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4024         };
4025     }
4026
4027     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4028 }
4029
4030 my $fast_plug_option = {
4031     'lock' => 1,
4032     'name' => 1,
4033     'onboot' => 1,
4034     'shares' => 1,
4035     'startup' => 1,
4036     'description' => 1,
4037     'protection' => 1,
4038 };
4039
4040 # hotplug changes in [PENDING]
4041 # $selection hash can be used to only apply specified options, for
4042 # example: { cores => 1 } (only apply changed 'cores')
4043 # $errors ref is used to return error messages
4044 sub vmconfig_hotplug_pending {
4045     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4046
4047     my $defaults = load_defaults();
4048
4049     # commit values which do not have any impact on running VM first
4050     # Note: those option cannot raise errors, we we do not care about
4051     # $selection and always apply them.
4052
4053     my $add_error = sub {
4054         my ($opt, $msg) = @_;
4055         $errors->{$opt} = "hotplug problem - $msg";
4056     };
4057
4058     my $changes = 0;
4059     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4060         if ($fast_plug_option->{$opt}) {
4061             $conf->{$opt} = $conf->{pending}->{$opt};
4062             delete $conf->{pending}->{$opt};
4063             $changes = 1;
4064         }
4065     }
4066
4067     if ($changes) {
4068         PVE::QemuConfig->write_config($vmid, $conf);
4069         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4070     }
4071
4072     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4073
4074     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4075     while (my ($opt, $force) = each %$pending_delete_hash) {
4076         next if $selection && !$selection->{$opt};
4077         eval {
4078             if ($opt eq 'hotplug') {
4079                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4080             } elsif ($opt eq 'tablet') {
4081                 die "skip\n" if !$hotplug_features->{usb};
4082                 if ($defaults->{tablet}) {
4083                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4084                 } else {
4085                     vm_deviceunplug($vmid, $conf, $opt);
4086                 }
4087             } elsif ($opt =~ m/^usb\d+/) {
4088                 die "skip\n";
4089                 # since we cannot reliably hot unplug usb devices
4090                 # we are disabling it
4091                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4092                 vm_deviceunplug($vmid, $conf, $opt);
4093             } elsif ($opt eq 'vcpus') {
4094                 die "skip\n" if !$hotplug_features->{cpu};
4095                 qemu_cpu_hotplug($vmid, $conf, undef);
4096             } elsif ($opt eq 'balloon') {
4097                 # enable balloon device is not hotpluggable
4098                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4099             } elsif ($fast_plug_option->{$opt}) {
4100                 # do nothing
4101             } elsif ($opt =~ m/^net(\d+)$/) {
4102                 die "skip\n" if !$hotplug_features->{network};
4103                 vm_deviceunplug($vmid, $conf, $opt);
4104             } elsif (is_valid_drivename($opt)) {
4105                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4106                 vm_deviceunplug($vmid, $conf, $opt);
4107                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4108             } elsif ($opt =~ m/^memory$/) {
4109                 die "skip\n" if !$hotplug_features->{memory};
4110                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4111             } elsif ($opt eq 'cpuunits') {
4112                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4113             } elsif ($opt eq 'cpulimit') {
4114                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4115             } else {
4116                 die "skip\n";
4117             }
4118         };
4119         if (my $err = $@) {
4120             &$add_error($opt, $err) if $err ne "skip\n";
4121         } else {
4122             # save new config if hotplug was successful
4123             delete $conf->{$opt};
4124             vmconfig_undelete_pending_option($conf, $opt);
4125             PVE::QemuConfig->write_config($vmid, $conf);
4126             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4127         }
4128     }
4129
4130     foreach my $opt (keys %{$conf->{pending}}) {
4131         next if $selection && !$selection->{$opt};
4132         my $value = $conf->{pending}->{$opt};
4133         eval {
4134             if ($opt eq 'hotplug') {
4135                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4136             } elsif ($opt eq 'tablet') {
4137                 die "skip\n" if !$hotplug_features->{usb};
4138                 if ($value == 1) {
4139                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4140                 } elsif ($value == 0) {
4141                     vm_deviceunplug($vmid, $conf, $opt);
4142                 }
4143             } elsif ($opt =~ m/^usb\d+$/) {
4144                 die "skip\n";
4145                 # since we cannot reliably hot unplug usb devices
4146                 # we are disabling it
4147                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4148                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4149                 die "skip\n" if !$d;
4150                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4151             } elsif ($opt eq 'vcpus') {
4152                 die "skip\n" if !$hotplug_features->{cpu};
4153                 qemu_cpu_hotplug($vmid, $conf, $value);
4154             } elsif ($opt eq 'balloon') {
4155                 # enable/disable balloning device is not hotpluggable
4156                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4157                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4158                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4159
4160                 # allow manual ballooning if shares is set to zero
4161                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4162                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4163                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4164                 }
4165             } elsif ($opt =~ m/^net(\d+)$/) {
4166                 # some changes can be done without hotplug
4167                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4168                                     $vmid, $opt, $value);
4169             } elsif (is_valid_drivename($opt)) {
4170                 # some changes can be done without hotplug
4171                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4172                                      $vmid, $opt, $value, 1);
4173             } elsif ($opt =~ m/^memory$/) { #dimms
4174                 die "skip\n" if !$hotplug_features->{memory};
4175                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4176             } elsif ($opt eq 'cpuunits') {
4177                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4178             } elsif ($opt eq 'cpulimit') {
4179                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4180                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4181             } else {
4182                 die "skip\n";  # skip non-hot-pluggable options
4183             }
4184         };
4185         if (my $err = $@) {
4186             &$add_error($opt, $err) if $err ne "skip\n";
4187         } else {
4188             # save new config if hotplug was successful
4189             $conf->{$opt} = $value;
4190             delete $conf->{pending}->{$opt};
4191             PVE::QemuConfig->write_config($vmid, $conf);
4192             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4193         }
4194     }
4195 }
4196
4197 sub try_deallocate_drive {
4198     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4199
4200     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4201         my $volid = $drive->{file};
4202         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4203             my $sid = PVE::Storage::parse_volume_id($volid);
4204             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4205
4206             # check if the disk is really unused
4207             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4208                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4209             PVE::Storage::vdisk_free($storecfg, $volid);
4210             return 1;
4211         } else {
4212             # If vm is not owner of this disk remove from config
4213             return 1;
4214         }
4215     }
4216
4217     return undef;
4218 }
4219
4220 sub vmconfig_delete_or_detach_drive {
4221     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4222
4223     my $drive = parse_drive($opt, $conf->{$opt});
4224
4225     my $rpcenv = PVE::RPCEnvironment::get();
4226     my $authuser = $rpcenv->get_user();
4227
4228     if ($force) {
4229         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4230         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4231     } else {
4232         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4233     }
4234 }
4235
4236 sub vmconfig_apply_pending {
4237     my ($vmid, $conf, $storecfg) = @_;
4238
4239     # cold plug
4240
4241     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4242     while (my ($opt, $force) = each %$pending_delete_hash) {
4243         die "internal error" if $opt =~ m/^unused/;
4244         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4245         if (!defined($conf->{$opt})) {
4246             vmconfig_undelete_pending_option($conf, $opt);
4247             PVE::QemuConfig->write_config($vmid, $conf);
4248         } elsif (is_valid_drivename($opt)) {
4249             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4250             vmconfig_undelete_pending_option($conf, $opt);
4251             delete $conf->{$opt};
4252             PVE::QemuConfig->write_config($vmid, $conf);
4253         } else {
4254             vmconfig_undelete_pending_option($conf, $opt);
4255             delete $conf->{$opt};
4256             PVE::QemuConfig->write_config($vmid, $conf);
4257         }
4258     }
4259
4260     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4261
4262     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4263         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4264
4265         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4266             # skip if nothing changed
4267         } elsif (is_valid_drivename($opt)) {
4268             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4269                 if defined($conf->{$opt});
4270             $conf->{$opt} = $conf->{pending}->{$opt};
4271         } else {
4272             $conf->{$opt} = $conf->{pending}->{$opt};
4273         }
4274
4275         delete $conf->{pending}->{$opt};
4276         PVE::QemuConfig->write_config($vmid, $conf);
4277     }
4278 }
4279
4280 my $safe_num_ne = sub {
4281     my ($a, $b) = @_;
4282
4283     return 0 if !defined($a) && !defined($b);
4284     return 1 if !defined($a);
4285     return 1 if !defined($b);
4286
4287     return $a != $b;
4288 };
4289
4290 my $safe_string_ne = sub {
4291     my ($a, $b) = @_;
4292
4293     return 0 if !defined($a) && !defined($b);
4294     return 1 if !defined($a);
4295     return 1 if !defined($b);
4296
4297     return $a ne $b;
4298 };
4299
4300 sub vmconfig_update_net {
4301     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4302
4303     my $newnet = parse_net($value);
4304
4305     if ($conf->{$opt}) {
4306         my $oldnet = parse_net($conf->{$opt});
4307
4308         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4309             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4310             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4311             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4312
4313             # for non online change, we try to hot-unplug
4314             die "skip\n" if !$hotplug;
4315             vm_deviceunplug($vmid, $conf, $opt);
4316         } else {
4317
4318             die "internal error" if $opt !~ m/net(\d+)/;
4319             my $iface = "tap${vmid}i$1";
4320
4321             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4322                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4323                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4324                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4325                 PVE::Network::tap_unplug($iface);
4326                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4327             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4328                 # Rate can be applied on its own but any change above needs to
4329                 # include the rate in tap_plug since OVS resets everything.
4330                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4331             }
4332
4333             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4334                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4335             }
4336
4337             return 1;
4338         }
4339     }
4340
4341     if ($hotplug) {
4342         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4343     } else {
4344         die "skip\n";
4345     }
4346 }
4347
4348 sub vmconfig_update_disk {
4349     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4350
4351     # fixme: do we need force?
4352
4353     my $drive = parse_drive($opt, $value);
4354
4355     if ($conf->{$opt}) {
4356
4357         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4358
4359             my $media = $drive->{media} || 'disk';
4360             my $oldmedia = $old_drive->{media} || 'disk';
4361             die "unable to change media type\n" if $media ne $oldmedia;
4362
4363             if (!drive_is_cdrom($old_drive)) {
4364
4365                 if ($drive->{file} ne $old_drive->{file}) {
4366
4367                     die "skip\n" if !$hotplug;
4368
4369                     # unplug and register as unused
4370                     vm_deviceunplug($vmid, $conf, $opt);
4371                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4372
4373                 } else {
4374                     # update existing disk
4375
4376                     # skip non hotpluggable value
4377                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4378                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4379                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4380                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4381                         die "skip\n";
4382                     }
4383
4384                     # apply throttle
4385                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4386                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4387                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4388                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4389                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4390                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4391                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4392                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4393                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4394                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4395                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4396                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4397
4398                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4399                                                    ($drive->{mbps} || 0)*1024*1024,
4400                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4401                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4402                                                    $drive->{iops} || 0,
4403                                                    $drive->{iops_rd} || 0,
4404                                                    $drive->{iops_wr} || 0,
4405                                                    ($drive->{mbps_max} || 0)*1024*1024,
4406                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4407                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4408                                                    $drive->{iops_max} || 0,
4409                                                    $drive->{iops_rd_max} || 0,
4410                                                    $drive->{iops_wr_max} || 0);
4411
4412                     }
4413
4414                     return 1;
4415                 }
4416
4417             } else { # cdrom
4418
4419                 if ($drive->{file} eq 'none') {
4420                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4421                 } else {
4422                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4423                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4424                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4425                 }
4426
4427                 return 1;
4428             }
4429         }
4430     }
4431
4432     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4433     # hotplug new disks
4434     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4435     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4436 }
4437
4438 sub vm_start {
4439     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4440         $forcemachine, $spice_ticket) = @_;
4441
4442     PVE::QemuConfig->lock_config($vmid, sub {
4443         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4444
4445         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4446
4447         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4448
4449         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4450
4451         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4452             vmconfig_apply_pending($vmid, $conf, $storecfg);
4453             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4454         }
4455
4456         my $defaults = load_defaults();
4457
4458         # set environment variable useful inside network script
4459         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4460
4461         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4462
4463         my $migrate_port = 0;
4464         my $migrate_uri;
4465         if ($statefile) {
4466             if ($statefile eq 'tcp') {
4467                 my $localip = "localhost";
4468                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4469                 my $nodename = PVE::INotify::nodename();
4470                 if ($datacenterconf->{migration_unsecure}) {
4471                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4472                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4473                 }
4474                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4475                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4476                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4477                 push @$cmd, '-incoming', $migrate_uri;
4478                 push @$cmd, '-S';
4479
4480             } elsif ($statefile eq 'unix') {
4481                 # should be default for secure migrations as a ssh TCP forward
4482                 # tunnel is not deterministic reliable ready and fails regurarly
4483                 # to set up in time, so use UNIX socket forwards
4484                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4485                 unlink $socket_addr;
4486
4487                 $migrate_uri = "unix:$socket_addr";
4488
4489                 push @$cmd, '-incoming', $migrate_uri;
4490                 push @$cmd, '-S';
4491
4492             } else {
4493                 push @$cmd, '-loadstate', $statefile;
4494             }
4495         } elsif ($paused) {
4496             push @$cmd, '-S';
4497         }
4498
4499         # host pci devices
4500         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4501           my $d = parse_hostpci($conf->{"hostpci$i"});
4502           next if !$d;
4503           my $pcidevices = $d->{pciid};
4504           foreach my $pcidevice (@$pcidevices) {
4505                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4506
4507                 my $info = pci_device_info("0000:$pciid");
4508                 die "IOMMU not present\n" if !check_iommu_support();
4509                 die "no pci device info for device '$pciid'\n" if !$info;
4510                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4511                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4512           }
4513         }
4514
4515         PVE::Storage::activate_volumes($storecfg, $vollist);
4516
4517         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4518             my $cmd = [];
4519             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4520             eval  { run_command($cmd); };
4521         }
4522
4523         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4524                                                   : $defaults->{cpuunits};
4525
4526         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4527
4528         my %properties = (
4529             Slice => 'qemu.slice',
4530             KillMode => 'none',
4531             CPUShares => $cpuunits
4532         );
4533
4534         if (my $cpulimit = $conf->{cpulimit}) {
4535             $properties{CPUQuota} = int($cpulimit * 100);
4536         }
4537         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4538
4539         if ($conf->{hugepages}) {
4540
4541             my $code = sub {
4542                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4543                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4544
4545                 PVE::QemuServer::Memory::hugepages_mount();
4546                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4547
4548                 eval  {
4549                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4550                     run_command($cmd, %run_params);
4551                 };
4552
4553                 if (my $err = $@) {
4554                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4555                     die $err;
4556                 }
4557
4558                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4559             };
4560             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4561
4562         } else {
4563             eval  {
4564                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4565                 run_command($cmd, %run_params);
4566             };
4567         }
4568
4569         if (my $err = $@) {
4570             # deactivate volumes if start fails
4571             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4572             die "start failed: $err";
4573         }
4574
4575         print "migration listens on $migrate_uri\n" if $migrate_uri;
4576
4577         if ($statefile && $statefile ne 'tcp')  {
4578             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4579             warn $@ if $@;
4580         }
4581
4582         if ($migratedfrom) {
4583
4584             eval {
4585                 set_migration_caps($vmid);
4586             };
4587             warn $@ if $@;
4588
4589             if ($spice_port) {
4590                 print "spice listens on port $spice_port\n";
4591                 if ($spice_ticket) {
4592                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4593                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4594                 }
4595             }
4596
4597         } else {
4598
4599             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4600                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4601                     if $conf->{balloon};
4602             }
4603
4604             foreach my $opt (keys %$conf) {
4605                 next if $opt !~  m/^net\d+$/;
4606                 my $nicconf = parse_net($conf->{$opt});
4607                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4608             }
4609         }
4610
4611         vm_mon_cmd_nocheck($vmid, 'qom-set',
4612                     path => "machine/peripheral/balloon0",
4613                     property => "guest-stats-polling-interval",
4614                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4615
4616     });
4617 }
4618
4619 sub vm_mon_cmd {
4620     my ($vmid, $execute, %params) = @_;
4621
4622     my $cmd = { execute => $execute, arguments => \%params };
4623     vm_qmp_command($vmid, $cmd);
4624 }
4625
4626 sub vm_mon_cmd_nocheck {
4627     my ($vmid, $execute, %params) = @_;
4628
4629     my $cmd = { execute => $execute, arguments => \%params };
4630     vm_qmp_command($vmid, $cmd, 1);
4631 }
4632
4633 sub vm_qmp_command {
4634     my ($vmid, $cmd, $nocheck) = @_;
4635
4636     my $res;
4637
4638     my $timeout;
4639     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4640         $timeout = $cmd->{arguments}->{timeout};
4641         delete $cmd->{arguments}->{timeout};
4642     }
4643
4644     eval {
4645         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4646         my $sname = qmp_socket($vmid);
4647         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4648             my $qmpclient = PVE::QMPClient->new();
4649
4650             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4651         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4652             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4653                 if scalar(%{$cmd->{arguments}});
4654             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4655         } else {
4656             die "unable to open monitor socket\n";
4657         }
4658     };
4659     if (my $err = $@) {
4660         syslog("err", "VM $vmid qmp command failed - $err");
4661         die $err;
4662     }
4663
4664     return $res;
4665 }
4666
4667 sub vm_human_monitor_command {
4668     my ($vmid, $cmdline) = @_;
4669
4670     my $res;
4671
4672     my $cmd = {
4673         execute => 'human-monitor-command',
4674         arguments => { 'command-line' => $cmdline},
4675     };
4676
4677     return vm_qmp_command($vmid, $cmd);
4678 }
4679
4680 sub vm_commandline {
4681     my ($storecfg, $vmid) = @_;
4682
4683     my $conf = PVE::QemuConfig->load_config($vmid);
4684
4685     my $defaults = load_defaults();
4686
4687     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4688
4689     return PVE::Tools::cmd2string($cmd);
4690 }
4691
4692 sub vm_reset {
4693     my ($vmid, $skiplock) = @_;
4694
4695     PVE::QemuConfig->lock_config($vmid, sub {
4696
4697         my $conf = PVE::QemuConfig->load_config($vmid);
4698
4699         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4700
4701         vm_mon_cmd($vmid, "system_reset");
4702     });
4703 }
4704
4705 sub get_vm_volumes {
4706     my ($conf) = @_;
4707
4708     my $vollist = [];
4709     foreach_volid($conf, sub {
4710         my ($volid, $is_cdrom) = @_;
4711
4712         return if $volid =~ m|^/|;
4713
4714         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4715         return if !$sid;
4716
4717         push @$vollist, $volid;
4718     });
4719
4720     return $vollist;
4721 }
4722
4723 sub vm_stop_cleanup {
4724     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4725
4726     eval {
4727
4728         if (!$keepActive) {
4729             my $vollist = get_vm_volumes($conf);
4730             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4731         }
4732
4733         foreach my $ext (qw(mon qmp pid vnc qga)) {
4734             unlink "/var/run/qemu-server/${vmid}.$ext";
4735         }
4736
4737         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4738     };
4739     warn $@ if $@; # avoid errors - just warn
4740 }
4741
4742 # Note: use $nockeck to skip tests if VM configuration file exists.
4743 # We need that when migration VMs to other nodes (files already moved)
4744 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4745 sub vm_stop {
4746     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4747
4748     $force = 1 if !defined($force) && !$shutdown;
4749
4750     if ($migratedfrom){
4751         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4752         kill 15, $pid if $pid;
4753         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4754         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4755         return;
4756     }
4757
4758     PVE::QemuConfig->lock_config($vmid, sub {
4759
4760         my $pid = check_running($vmid, $nocheck);
4761         return if !$pid;
4762
4763         my $conf;
4764         if (!$nocheck) {
4765             $conf = PVE::QemuConfig->load_config($vmid);
4766             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4767             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4768                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4769                 $timeout = $opts->{down} if $opts->{down};
4770             }
4771         }
4772
4773         $timeout = 60 if !defined($timeout);
4774
4775         eval {
4776             if ($shutdown) {
4777                 if (defined($conf) && $conf->{agent}) {
4778                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4779                 } else {
4780                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4781                 }
4782             } else {
4783                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4784             }
4785         };
4786         my $err = $@;
4787
4788         if (!$err) {
4789             my $count = 0;
4790             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4791                 $count++;
4792                 sleep 1;
4793             }
4794
4795             if ($count >= $timeout) {
4796                 if ($force) {
4797                     warn "VM still running - terminating now with SIGTERM\n";
4798                     kill 15, $pid;
4799                 } else {
4800                     die "VM quit/powerdown failed - got timeout\n";
4801                 }
4802             } else {
4803                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4804                 return;
4805             }
4806         } else {
4807             if ($force) {
4808                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4809                 kill 15, $pid;
4810             } else {
4811                 die "VM quit/powerdown failed\n";
4812             }
4813         }
4814
4815         # wait again
4816         $timeout = 10;
4817
4818         my $count = 0;
4819         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4820             $count++;
4821             sleep 1;
4822         }
4823
4824         if ($count >= $timeout) {
4825             warn "VM still running - terminating now with SIGKILL\n";
4826             kill 9, $pid;
4827             sleep 1;
4828         }
4829
4830         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4831    });
4832 }
4833
4834 sub vm_suspend {
4835     my ($vmid, $skiplock) = @_;
4836
4837     PVE::QemuConfig->lock_config($vmid, sub {
4838
4839         my $conf = PVE::QemuConfig->load_config($vmid);
4840
4841         PVE::QemuConfig->check_lock($conf)
4842             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4843
4844         vm_mon_cmd($vmid, "stop");
4845     });
4846 }
4847
4848 sub vm_resume {
4849     my ($vmid, $skiplock, $nocheck) = @_;
4850
4851     PVE::QemuConfig->lock_config($vmid, sub {
4852
4853         if (!$nocheck) {
4854
4855             my $conf = PVE::QemuConfig->load_config($vmid);
4856
4857             PVE::QemuConfig->check_lock($conf)
4858                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4859
4860             vm_mon_cmd($vmid, "cont");
4861
4862         } else {
4863             vm_mon_cmd_nocheck($vmid, "cont");
4864         }
4865     });
4866 }
4867
4868 sub vm_sendkey {
4869     my ($vmid, $skiplock, $key) = @_;
4870
4871     PVE::QemuConfig->lock_config($vmid, sub {
4872
4873         my $conf = PVE::QemuConfig->load_config($vmid);
4874
4875         # there is no qmp command, so we use the human monitor command
4876         vm_human_monitor_command($vmid, "sendkey $key");
4877     });
4878 }
4879
4880 sub vm_destroy {
4881     my ($storecfg, $vmid, $skiplock) = @_;
4882
4883     PVE::QemuConfig->lock_config($vmid, sub {
4884
4885         my $conf = PVE::QemuConfig->load_config($vmid);
4886
4887         if (!check_running($vmid)) {
4888             destroy_vm($storecfg, $vmid, undef, $skiplock);
4889         } else {
4890             die "VM $vmid is running - destroy failed\n";
4891         }
4892     });
4893 }
4894
4895 # pci helpers
4896
4897 sub file_write {
4898     my ($filename, $buf) = @_;
4899
4900     my $fh = IO::File->new($filename, "w");
4901     return undef if !$fh;
4902
4903     my $res = print $fh $buf;
4904
4905     $fh->close();
4906
4907     return $res;
4908 }
4909
4910 sub pci_device_info {
4911     my ($name) = @_;
4912
4913     my $res;
4914
4915     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4916     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4917
4918     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4919     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4920
4921     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4922     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4923
4924     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4925     return undef if !defined($product) || $product !~ s/^0x//;
4926
4927     $res = {
4928         name => $name,
4929         vendor => $vendor,
4930         product => $product,
4931         domain => $domain,
4932         bus => $bus,
4933         slot => $slot,
4934         func => $func,
4935         irq => $irq,
4936         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4937     };
4938
4939     return $res;
4940 }
4941
4942 sub pci_dev_reset {
4943     my ($dev) = @_;
4944
4945     my $name = $dev->{name};
4946
4947     my $fn = "$pcisysfs/devices/$name/reset";
4948
4949     return file_write($fn, "1");
4950 }
4951
4952 sub pci_dev_bind_to_vfio {
4953     my ($dev) = @_;
4954
4955     my $name = $dev->{name};
4956
4957     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4958
4959     if (!-d $vfio_basedir) {
4960         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4961     }
4962     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4963
4964     my $testdir = "$vfio_basedir/$name";
4965     return 1 if -d $testdir;
4966
4967     my $data = "$dev->{vendor} $dev->{product}";
4968     return undef if !file_write("$vfio_basedir/new_id", $data);
4969
4970     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4971     if (!file_write($fn, $name)) {
4972         return undef if -f $fn;
4973     }
4974
4975     $fn = "$vfio_basedir/bind";
4976     if (! -d $testdir) {
4977         return undef if !file_write($fn, $name);
4978     }
4979
4980     return -d $testdir;
4981 }
4982
4983 sub pci_dev_group_bind_to_vfio {
4984     my ($pciid) = @_;
4985
4986     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4987
4988     if (!-d $vfio_basedir) {
4989         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4990     }
4991     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4992
4993     # get IOMMU group devices
4994     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4995       my @devs = grep /^0000:/, readdir($D);
4996     closedir($D);
4997
4998     foreach my $pciid (@devs) {
4999         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5000
5001         # pci bridges, switches or root ports are not supported
5002         # they have a pci_bus subdirectory so skip them
5003         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5004
5005         my $info = pci_device_info($1);
5006         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5007     }
5008
5009     return 1;
5010 }
5011
5012 # vzdump restore implementaion
5013
5014 sub tar_archive_read_firstfile {
5015     my $archive = shift;
5016
5017     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5018
5019     # try to detect archive type first
5020     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5021         die "unable to open file '$archive'\n";
5022     my $firstfile = <$fh>;
5023     kill 15, $pid;
5024     close $fh;
5025
5026     die "ERROR: archive contaions no data\n" if !$firstfile;
5027     chomp $firstfile;
5028
5029     return $firstfile;
5030 }
5031
5032 sub tar_restore_cleanup {
5033     my ($storecfg, $statfile) = @_;
5034
5035     print STDERR "starting cleanup\n";
5036
5037     if (my $fd = IO::File->new($statfile, "r")) {
5038         while (defined(my $line = <$fd>)) {
5039             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5040                 my $volid = $2;
5041                 eval {
5042                     if ($volid =~ m|^/|) {
5043                         unlink $volid || die 'unlink failed\n';
5044                     } else {
5045                         PVE::Storage::vdisk_free($storecfg, $volid);
5046                     }
5047                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5048                 };
5049                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5050             } else {
5051                 print STDERR "unable to parse line in statfile - $line";
5052             }
5053         }
5054         $fd->close();
5055     }
5056 }
5057
5058 sub restore_archive {
5059     my ($archive, $vmid, $user, $opts) = @_;
5060
5061     my $format = $opts->{format};
5062     my $comp;
5063
5064     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5065         $format = 'tar' if !$format;
5066         $comp = 'gzip';
5067     } elsif ($archive =~ m/\.tar$/) {
5068         $format = 'tar' if !$format;
5069     } elsif ($archive =~ m/.tar.lzo$/) {
5070         $format = 'tar' if !$format;
5071         $comp = 'lzop';
5072     } elsif ($archive =~ m/\.vma$/) {
5073         $format = 'vma' if !$format;
5074     } elsif ($archive =~ m/\.vma\.gz$/) {
5075         $format = 'vma' if !$format;
5076         $comp = 'gzip';
5077     } elsif ($archive =~ m/\.vma\.lzo$/) {
5078         $format = 'vma' if !$format;
5079         $comp = 'lzop';
5080     } else {
5081         $format = 'vma' if !$format; # default
5082     }
5083
5084     # try to detect archive format
5085     if ($format eq 'tar') {
5086         return restore_tar_archive($archive, $vmid, $user, $opts);
5087     } else {
5088         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5089     }
5090 }
5091
5092 sub restore_update_config_line {
5093     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5094
5095     return if $line =~ m/^\#qmdump\#/;
5096     return if $line =~ m/^\#vzdump\#/;
5097     return if $line =~ m/^lock:/;
5098     return if $line =~ m/^unused\d+:/;
5099     return if $line =~ m/^parent:/;
5100     return if $line =~ m/^template:/; # restored VM is never a template
5101
5102     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5103     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5104         # try to convert old 1.X settings
5105         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5106         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5107             my ($model, $macaddr) = split(/\=/, $devconfig);
5108             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5109             my $net = {
5110                 model => $model,
5111                 bridge => "vmbr$ind",
5112                 macaddr => $macaddr,
5113             };
5114             my $netstr = print_net($net);
5115
5116             print $outfd "net$cookie->{netcount}: $netstr\n";
5117             $cookie->{netcount}++;
5118         }
5119     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5120         my ($id, $netstr) = ($1, $2);
5121         my $net = parse_net($netstr);
5122         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5123         $netstr = print_net($net);
5124         print $outfd "$id: $netstr\n";
5125     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5126         my $virtdev = $1;
5127         my $value = $3;
5128         my $di = parse_drive($virtdev, $value);
5129         if (defined($di->{backup}) && !$di->{backup}) {
5130             print $outfd "#$line";
5131         } elsif ($map->{$virtdev}) {
5132             delete $di->{format}; # format can change on restore
5133             $di->{file} = $map->{$virtdev};
5134             $value = print_drive($vmid, $di);
5135             print $outfd "$virtdev: $value\n";
5136         } else {
5137             print $outfd $line;
5138         }
5139     } else {
5140         print $outfd $line;
5141     }
5142 }
5143
5144 sub scan_volids {
5145     my ($cfg, $vmid) = @_;
5146
5147     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5148
5149     my $volid_hash = {};
5150     foreach my $storeid (keys %$info) {
5151         foreach my $item (@{$info->{$storeid}}) {
5152             next if !($item->{volid} && $item->{size});
5153             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5154             $volid_hash->{$item->{volid}} = $item;
5155         }
5156     }
5157
5158     return $volid_hash;
5159 }
5160
5161 sub is_volume_in_use {
5162     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5163
5164     my $path = PVE::Storage::path($storecfg, $volid);
5165
5166     my $scan_config = sub {
5167         my ($cref, $snapname) = @_;
5168
5169         foreach my $key (keys %$cref) {
5170             my $value = $cref->{$key};
5171             if (is_valid_drivename($key)) {
5172                 next if $skip_drive && $key eq $skip_drive;
5173                 my $drive = parse_drive($key, $value);
5174                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5175                 return 1 if $volid eq $drive->{file};
5176                 if ($drive->{file} =~ m!^/!) {
5177                     return 1 if $drive->{file} eq $path;
5178                 } else {
5179                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5180                     next if !$storeid;
5181                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5182                     next if !$scfg;
5183                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5184                 }
5185             }
5186         }
5187
5188         return 0;
5189     };
5190
5191     return 1 if &$scan_config($conf);
5192
5193     undef $skip_drive;
5194
5195     foreach my $snapname (keys %{$conf->{snapshots}}) {
5196         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5197     }
5198
5199     return 0;
5200 }
5201
5202 sub update_disksize {
5203     my ($vmid, $conf, $volid_hash) = @_;
5204
5205     my $changes;
5206
5207     my $used = {};
5208
5209     # Note: it is allowed to define multiple storages with same path (alias), so
5210     # we need to check both 'volid' and real 'path' (two different volid can point
5211     # to the same path).
5212
5213     my $usedpath = {};
5214
5215     # update size info
5216     foreach my $opt (keys %$conf) {
5217         if (is_valid_drivename($opt)) {
5218             my $drive = parse_drive($opt, $conf->{$opt});
5219             my $volid = $drive->{file};
5220             next if !$volid;
5221
5222             $used->{$volid} = 1;
5223             if ($volid_hash->{$volid} &&
5224                 (my $path = $volid_hash->{$volid}->{path})) {
5225                 $usedpath->{$path} = 1;
5226             }
5227
5228             next if drive_is_cdrom($drive);
5229             next if !$volid_hash->{$volid};
5230
5231             $drive->{size} = $volid_hash->{$volid}->{size};
5232             my $new = print_drive($vmid, $drive);
5233             if ($new ne $conf->{$opt}) {
5234                 $changes = 1;
5235                 $conf->{$opt} = $new;
5236             }
5237         }
5238     }
5239
5240     # remove 'unusedX' entry if volume is used
5241     foreach my $opt (keys %$conf) {
5242         next if $opt !~ m/^unused\d+$/;
5243         my $volid = $conf->{$opt};
5244         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5245         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5246             $changes = 1;
5247             delete $conf->{$opt};
5248         }
5249     }
5250
5251     foreach my $volid (sort keys %$volid_hash) {
5252         next if $volid =~ m/vm-$vmid-state-/;
5253         next if $used->{$volid};
5254         my $path = $volid_hash->{$volid}->{path};
5255         next if !$path; # just to be sure
5256         next if $usedpath->{$path};
5257         $changes = 1;
5258         PVE::QemuConfig->add_unused_volume($conf, $volid);
5259         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5260     }
5261
5262     return $changes;
5263 }
5264
5265 sub rescan {
5266     my ($vmid, $nolock) = @_;
5267
5268     my $cfg = PVE::Storage::config();
5269
5270     my $volid_hash = scan_volids($cfg, $vmid);
5271
5272     my $updatefn =  sub {
5273         my ($vmid) = @_;
5274
5275         my $conf = PVE::QemuConfig->load_config($vmid);
5276
5277         PVE::QemuConfig->check_lock($conf);
5278
5279         my $vm_volids = {};
5280         foreach my $volid (keys %$volid_hash) {
5281             my $info = $volid_hash->{$volid};
5282             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5283         }
5284
5285         my $changes = update_disksize($vmid, $conf, $vm_volids);
5286
5287         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5288     };
5289
5290     if (defined($vmid)) {
5291         if ($nolock) {
5292             &$updatefn($vmid);
5293         } else {
5294             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5295         }
5296     } else {
5297         my $vmlist = config_list();
5298         foreach my $vmid (keys %$vmlist) {
5299             if ($nolock) {
5300                 &$updatefn($vmid);
5301             } else {
5302                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5303             }
5304         }
5305     }
5306 }
5307
5308 sub restore_vma_archive {
5309     my ($archive, $vmid, $user, $opts, $comp) = @_;
5310
5311     my $input = $archive eq '-' ? "<&STDIN" : undef;
5312     my $readfrom = $archive;
5313
5314     my $uncomp = '';
5315     if ($comp) {
5316         $readfrom = '-';
5317         my $qarchive = PVE::Tools::shellquote($archive);
5318         if ($comp eq 'gzip') {
5319             $uncomp = "zcat $qarchive|";
5320         } elsif ($comp eq 'lzop') {
5321             $uncomp = "lzop -d -c $qarchive|";
5322         } else {
5323             die "unknown compression method '$comp'\n";
5324         }
5325
5326     }
5327
5328     my $tmpdir = "/var/tmp/vzdumptmp$$";
5329     rmtree $tmpdir;
5330
5331     # disable interrupts (always do cleanups)
5332     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5333         warn "got interrupt - ignored\n";
5334     };
5335
5336     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5337     POSIX::mkfifo($mapfifo, 0600);
5338     my $fifofh;
5339
5340     my $openfifo = sub {
5341         open($fifofh, '>', $mapfifo) || die $!;
5342     };
5343
5344     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5345
5346     my $oldtimeout;
5347     my $timeout = 5;
5348
5349     my $devinfo = {};
5350
5351     my $rpcenv = PVE::RPCEnvironment::get();
5352
5353     my $conffile = PVE::QemuConfig->config_file($vmid);
5354     my $tmpfn = "$conffile.$$.tmp";
5355
5356     # Note: $oldconf is undef if VM does not exists
5357     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5358     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5359
5360     my $print_devmap = sub {
5361         my $virtdev_hash = {};
5362
5363         my $cfgfn = "$tmpdir/qemu-server.conf";
5364
5365         # we can read the config - that is already extracted
5366         my $fh = IO::File->new($cfgfn, "r") ||
5367             "unable to read qemu-server.conf - $!\n";
5368
5369         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5370         if (-f $fwcfgfn) {
5371             my $pve_firewall_dir = '/etc/pve/firewall';
5372             mkdir $pve_firewall_dir; # make sure the dir exists
5373             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5374         }
5375
5376         while (defined(my $line = <$fh>)) {
5377             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5378                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5379                 die "archive does not contain data for drive '$virtdev'\n"
5380                     if !$devinfo->{$devname};
5381                 if (defined($opts->{storage})) {
5382                     $storeid = $opts->{storage} || 'local';
5383                 } elsif (!$storeid) {
5384                     $storeid = 'local';
5385                 }
5386                 $format = 'raw' if !$format;
5387                 $devinfo->{$devname}->{devname} = $devname;
5388                 $devinfo->{$devname}->{virtdev} = $virtdev;
5389                 $devinfo->{$devname}->{format} = $format;
5390                 $devinfo->{$devname}->{storeid} = $storeid;
5391
5392                 # check permission on storage
5393                 my $pool = $opts->{pool}; # todo: do we need that?
5394                 if ($user ne 'root@pam') {
5395                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5396                 }
5397
5398                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5399             }
5400         }
5401
5402         foreach my $devname (keys %$devinfo) {
5403             die "found no device mapping information for device '$devname'\n"
5404                 if !$devinfo->{$devname}->{virtdev};
5405         }
5406
5407         my $cfg = PVE::Storage::config();
5408
5409         # create empty/temp config
5410         if ($oldconf) {
5411             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5412             foreach_drive($oldconf, sub {
5413                 my ($ds, $drive) = @_;
5414
5415                 return if drive_is_cdrom($drive);
5416
5417                 my $volid = $drive->{file};
5418
5419                 return if !$volid || $volid =~ m|^/|;
5420
5421                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5422                 return if !$path || !$owner || ($owner != $vmid);
5423
5424                 # Note: only delete disk we want to restore
5425                 # other volumes will become unused
5426                 if ($virtdev_hash->{$ds}) {
5427                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5428                     if (my $err = $@) {
5429                         warn $err;
5430                     }
5431                 }
5432             });
5433
5434             # delete vmstate files
5435             # since after the restore we have no snapshots anymore
5436             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5437                 my $snap = $oldconf->{snapshots}->{$snapname};
5438                 if ($snap->{vmstate}) {
5439                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5440                     if (my $err = $@) {
5441                         warn $err;
5442                     }
5443                 }
5444             }
5445         }
5446
5447         my $map = {};
5448         foreach my $virtdev (sort keys %$virtdev_hash) {
5449             my $d = $virtdev_hash->{$virtdev};
5450             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5451             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5452
5453             # test if requested format is supported
5454             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5455             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5456             $d->{format} = $defFormat if !$supported;
5457
5458             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5459                                                   $d->{format}, undef, $alloc_size);
5460             print STDERR "new volume ID is '$volid'\n";
5461             $d->{volid} = $volid;
5462             my $path = PVE::Storage::path($cfg, $volid);
5463
5464             PVE::Storage::activate_volumes($cfg,[$volid]);
5465
5466             my $write_zeros = 1;
5467             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5468                 $write_zeros = 0;
5469             }
5470
5471             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5472
5473             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5474             $map->{$virtdev} = $volid;
5475         }
5476
5477         $fh->seek(0, 0) || die "seek failed - $!\n";
5478
5479         my $outfd = new IO::File ($tmpfn, "w") ||
5480             die "unable to write config for VM $vmid\n";
5481
5482         my $cookie = { netcount => 0 };
5483         while (defined(my $line = <$fh>)) {
5484             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5485         }
5486
5487         $fh->close();
5488         $outfd->close();
5489     };
5490
5491     eval {
5492         # enable interrupts
5493         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5494             die "interrupted by signal\n";
5495         };
5496         local $SIG{ALRM} = sub { die "got timeout\n"; };
5497
5498         $oldtimeout = alarm($timeout);
5499
5500         my $parser = sub {
5501             my $line = shift;
5502
5503             print "$line\n";
5504
5505             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5506                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5507                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5508             } elsif ($line =~ m/^CTIME: /) {
5509                 # we correctly received the vma config, so we can disable
5510                 # the timeout now for disk allocation (set to 10 minutes, so
5511                 # that we always timeout if something goes wrong)
5512                 alarm(600);
5513                 &$print_devmap();
5514                 print $fifofh "done\n";
5515                 my $tmp = $oldtimeout || 0;
5516                 $oldtimeout = undef;
5517                 alarm($tmp);
5518                 close($fifofh);
5519             }
5520         };
5521
5522         print "restore vma archive: $cmd\n";
5523         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5524     };
5525     my $err = $@;
5526
5527     alarm($oldtimeout) if $oldtimeout;
5528
5529     my $vollist = [];
5530     foreach my $devname (keys %$devinfo) {
5531         my $volid = $devinfo->{$devname}->{volid};
5532         push @$vollist, $volid if $volid;
5533     }
5534
5535     my $cfg = PVE::Storage::config();
5536     PVE::Storage::deactivate_volumes($cfg, $vollist);
5537
5538     unlink $mapfifo;
5539
5540     if ($err) {
5541         rmtree $tmpdir;
5542         unlink $tmpfn;
5543
5544         foreach my $devname (keys %$devinfo) {
5545             my $volid = $devinfo->{$devname}->{volid};
5546             next if !$volid;
5547             eval {
5548                 if ($volid =~ m|^/|) {
5549                     unlink $volid || die 'unlink failed\n';
5550                 } else {
5551                     PVE::Storage::vdisk_free($cfg, $volid);
5552                 }
5553                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5554             };
5555             print STDERR "unable to cleanup '$volid' - $@" if $@;
5556         }
5557         die $err;
5558     }
5559
5560     rmtree $tmpdir;
5561
5562     rename($tmpfn, $conffile) ||
5563         die "unable to commit configuration file '$conffile'\n";
5564
5565     PVE::Cluster::cfs_update(); # make sure we read new file
5566
5567     eval { rescan($vmid, 1); };
5568     warn $@ if $@;
5569 }
5570
5571 sub restore_tar_archive {
5572     my ($archive, $vmid, $user, $opts) = @_;
5573
5574     if ($archive ne '-') {
5575         my $firstfile = tar_archive_read_firstfile($archive);
5576         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5577             if $firstfile ne 'qemu-server.conf';
5578     }
5579
5580     my $storecfg = PVE::Storage::config();
5581
5582     # destroy existing data - keep empty config
5583     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5584     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5585
5586     my $tocmd = "/usr/lib/qemu-server/qmextract";
5587
5588     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5589     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5590     $tocmd .= ' --prealloc' if $opts->{prealloc};
5591     $tocmd .= ' --info' if $opts->{info};
5592
5593     # tar option "xf" does not autodetect compression when read from STDIN,
5594     # so we pipe to zcat
5595     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5596         PVE::Tools::shellquote("--to-command=$tocmd");
5597
5598     my $tmpdir = "/var/tmp/vzdumptmp$$";
5599     mkpath $tmpdir;
5600
5601     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5602     local $ENV{VZDUMP_VMID} = $vmid;
5603     local $ENV{VZDUMP_USER} = $user;
5604
5605     my $conffile = PVE::QemuConfig->config_file($vmid);
5606     my $tmpfn = "$conffile.$$.tmp";
5607
5608     # disable interrupts (always do cleanups)
5609     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5610         print STDERR "got interrupt - ignored\n";
5611     };
5612
5613     eval {
5614         # enable interrupts
5615         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5616             die "interrupted by signal\n";
5617         };
5618
5619         if ($archive eq '-') {
5620             print "extracting archive from STDIN\n";
5621             run_command($cmd, input => "<&STDIN");
5622         } else {
5623             print "extracting archive '$archive'\n";
5624             run_command($cmd);
5625         }
5626
5627         return if $opts->{info};
5628
5629         # read new mapping
5630         my $map = {};
5631         my $statfile = "$tmpdir/qmrestore.stat";
5632         if (my $fd = IO::File->new($statfile, "r")) {
5633             while (defined (my $line = <$fd>)) {
5634                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5635                     $map->{$1} = $2 if $1;
5636                 } else {
5637                     print STDERR "unable to parse line in statfile - $line\n";
5638                 }
5639             }
5640             $fd->close();
5641         }
5642
5643         my $confsrc = "$tmpdir/qemu-server.conf";
5644
5645         my $srcfd = new IO::File($confsrc, "r") ||
5646             die "unable to open file '$confsrc'\n";
5647
5648         my $outfd = new IO::File ($tmpfn, "w") ||
5649             die "unable to write config for VM $vmid\n";
5650
5651         my $cookie = { netcount => 0 };
5652         while (defined (my $line = <$srcfd>)) {
5653             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5654         }
5655
5656         $srcfd->close();
5657         $outfd->close();
5658     };
5659     my $err = $@;
5660
5661     if ($err) {
5662
5663         unlink $tmpfn;
5664
5665         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5666
5667         die $err;
5668     }
5669
5670     rmtree $tmpdir;
5671
5672     rename $tmpfn, $conffile ||
5673         die "unable to commit configuration file '$conffile'\n";
5674
5675     PVE::Cluster::cfs_update(); # make sure we read new file
5676
5677     eval { rescan($vmid, 1); };
5678     warn $@ if $@;
5679 };
5680
5681 sub foreach_writable_storage {
5682     my ($conf, $func) = @_;
5683
5684     my $sidhash = {};
5685
5686     foreach my $ds (keys %$conf) {
5687         next if !is_valid_drivename($ds);
5688
5689         my $drive = parse_drive($ds, $conf->{$ds});
5690         next if !$drive;
5691         next if drive_is_cdrom($drive);
5692
5693         my $volid = $drive->{file};
5694
5695         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5696         $sidhash->{$sid} = $sid if $sid;
5697     }
5698
5699     foreach my $sid (sort keys %$sidhash) {
5700         &$func($sid);
5701     }
5702 }
5703
5704 sub do_snapshots_with_qemu {
5705     my ($storecfg, $volid) = @_;
5706
5707     my $storage_name = PVE::Storage::parse_volume_id($volid);
5708
5709     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5710         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5711         return 1;
5712     }
5713
5714     if ($volid =~ m/\.(qcow2|qed)$/){
5715         return 1;
5716     }
5717
5718     return undef;
5719 }
5720
5721 sub qga_check_running {
5722     my ($vmid) = @_;
5723
5724     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5725     if ($@) {
5726         warn "Qemu Guest Agent are not running - $@";
5727         return 0;
5728     }
5729     return 1;
5730 }
5731
5732 sub template_create {
5733     my ($vmid, $conf, $disk) = @_;
5734
5735     my $storecfg = PVE::Storage::config();
5736
5737     foreach_drive($conf, sub {
5738         my ($ds, $drive) = @_;
5739
5740         return if drive_is_cdrom($drive);
5741         return if $disk && $ds ne $disk;
5742
5743         my $volid = $drive->{file};
5744         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5745
5746         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5747         $drive->{file} = $voliddst;
5748         $conf->{$ds} = print_drive($vmid, $drive);
5749         PVE::QemuConfig->write_config($vmid, $conf);
5750     });
5751 }
5752
5753 sub qemu_img_convert {
5754     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5755
5756     my $storecfg = PVE::Storage::config();
5757     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5758     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5759
5760     if ($src_storeid && $dst_storeid) {
5761
5762         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5763
5764         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5765         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5766
5767         my $src_format = qemu_img_format($src_scfg, $src_volname);
5768         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5769
5770         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5771         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5772
5773         my $cmd = [];
5774         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5775         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5776         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5777         if ($is_zero_initialized) {
5778             push @$cmd, "zeroinit:$dst_path";
5779         } else {
5780             push @$cmd, $dst_path;
5781         }
5782
5783         my $parser = sub {
5784             my $line = shift;
5785             if($line =~ m/\((\S+)\/100\%\)/){
5786                 my $percent = $1;
5787                 my $transferred = int($size * $percent / 100);
5788                 my $remaining = $size - $transferred;
5789
5790                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5791             }
5792
5793         };
5794
5795         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5796         my $err = $@;
5797         die "copy failed: $err" if $err;
5798     }
5799 }
5800
5801 sub qemu_img_format {
5802     my ($scfg, $volname) = @_;
5803
5804     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5805         return $1;
5806     } else {
5807         return "raw";
5808     }
5809 }
5810
5811 sub qemu_drive_mirror {
5812     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5813
5814     my $storecfg = PVE::Storage::config();
5815     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5816
5817     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5818
5819     my $format = qemu_img_format($dst_scfg, $dst_volname);
5820
5821     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5822
5823     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5824
5825     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5826     $opts->{format} = $format if $format;
5827
5828     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5829
5830     my $finish_job = sub {
5831         while (1) {
5832             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5833             my $stat = @$stats[0];
5834             last if !$stat;
5835             sleep 1;
5836         }
5837     };
5838
5839     eval {
5840     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5841         while (1) {
5842             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5843             my $stat = @$stats[0];
5844             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5845             die "error job is not mirroring" if $stat->{type} ne "mirror";
5846
5847             my $busy = $stat->{busy};
5848             my $ready = $stat->{ready};
5849
5850             if (my $total = $stat->{len}) {
5851                 my $transferred = $stat->{offset} || 0;
5852                 my $remaining = $total - $transferred;
5853                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5854
5855                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5856             }
5857
5858
5859             if ($stat->{ready} eq 'true') {
5860
5861                 last if $vmiddst != $vmid;
5862
5863                 # try to switch the disk if source and destination are on the same guest
5864                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5865                 if (!$@) {
5866                     &$finish_job();
5867                     last;
5868                 }
5869                 die $@ if $@ !~ m/cannot be completed/;
5870             }
5871             sleep 1;
5872         }
5873
5874
5875     };
5876     my $err = $@;
5877
5878     my $cancel_job = sub {
5879         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5880         &$finish_job();
5881     };
5882
5883     if ($err) {
5884         eval { &$cancel_job(); };
5885         die "mirroring error: $err";
5886     }
5887
5888     if ($vmiddst != $vmid) {
5889         # if we clone a disk for a new target vm, we don't switch the disk
5890         &$cancel_job(); # so we call block-job-cancel
5891     }
5892 }
5893
5894 sub clone_disk {
5895     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5896         $newvmid, $storage, $format, $full, $newvollist) = @_;
5897
5898     my $newvolid;
5899
5900     if (!$full) {
5901         print "create linked clone of drive $drivename ($drive->{file})\n";
5902         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5903         push @$newvollist, $newvolid;
5904     } else {
5905         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5906         $storeid = $storage if $storage;
5907
5908         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5909         if (!$format) {
5910             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5911             $format = qemu_img_format($scfg, $volname);
5912         }
5913
5914         # test if requested format is supported - else use default
5915         my $supported = grep { $_ eq $format } @$validFormats;
5916         $format = $defFormat if !$supported;
5917
5918         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5919
5920         print "create full clone of drive $drivename ($drive->{file})\n";
5921         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5922         push @$newvollist, $newvolid;
5923
5924         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
5925
5926         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5927         if (!$running || $snapname) {
5928             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5929         } else {
5930             #qemu 2.6
5931             die "drive-mirror is not working currently when iothread is enabled" if $drive->{iothread};
5932
5933             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5934         }
5935     }
5936
5937     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5938
5939     my $disk = $drive;
5940     $disk->{format} = undef;
5941     $disk->{file} = $newvolid;
5942     $disk->{size} = $size;
5943
5944     return $disk;
5945 }
5946
5947 # this only works if VM is running
5948 sub get_current_qemu_machine {
5949     my ($vmid) = @_;
5950
5951     my $cmd = { execute => 'query-machines', arguments => {} };
5952     my $res = vm_qmp_command($vmid, $cmd);
5953
5954     my ($current, $default);
5955     foreach my $e (@$res) {
5956         $default = $e->{name} if $e->{'is-default'};
5957         $current = $e->{name} if $e->{'is-current'};
5958     }
5959
5960     # fallback to the default machine if current is not supported by qemu
5961     return $current || $default || 'pc';
5962 }
5963
5964 sub get_running_qemu_version {
5965     my ($vmid) = @_;
5966     my $cmd = { execute => 'query-version', arguments => {} };
5967     my $res = vm_qmp_command($vmid, $cmd);
5968     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
5969 }
5970
5971 sub qemu_machine_feature_enabled {
5972     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5973
5974     my $current_major;
5975     my $current_minor;
5976
5977     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5978
5979         $current_major = $3;
5980         $current_minor = $4;
5981
5982     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5983
5984         $current_major = $1;
5985         $current_minor = $2;
5986     }
5987
5988     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5989
5990
5991 }
5992
5993 sub qemu_machine_pxe {
5994     my ($vmid, $conf, $machine) = @_;
5995
5996     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5997
5998     foreach my $opt (keys %$conf) {
5999         next if $opt !~ m/^net(\d+)$/;
6000         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6001         next if !$net;
6002         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6003         return $machine.".pxe" if $romfile =~ m/pxe/;
6004         last;
6005     }
6006
6007     return $machine;
6008 }
6009
6010 sub qemu_use_old_bios_files {
6011     my ($machine_type) = @_;
6012
6013     return if !$machine_type;
6014
6015     my $use_old_bios_files = undef;
6016
6017     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6018         $machine_type = $1;
6019         $use_old_bios_files = 1;
6020     } else {
6021         my $kvmver = kvm_user_version();
6022         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6023         # load new efi bios files on migration. So this hack is required to allow
6024         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6025         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6026         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6027     }
6028
6029     return ($use_old_bios_files, $machine_type);
6030 }
6031
6032 sub lspci {
6033
6034     my $devices = {};
6035
6036     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6037             my (undef, $id, $function) = @_;
6038             my $res = { id => $id, function => $function};
6039             push @{$devices->{$id}}, $res;
6040     });
6041
6042     return $devices;
6043 }
6044
6045 sub vm_iothreads_list {
6046     my ($vmid) = @_;
6047
6048     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6049
6050     my $iothreads = {};
6051     foreach my $iothread (@$res) {
6052         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6053     }
6054
6055     return $iothreads;
6056 }
6057
6058 sub scsihw_infos {
6059     my ($conf, $drive) = @_;
6060
6061     my $maxdev = 0;
6062
6063     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6064         $maxdev = 7;
6065     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6066         $maxdev = 1;
6067     } else {
6068         $maxdev = 256;
6069     }
6070
6071     my $controller = int($drive->{index} / $maxdev);
6072     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6073
6074     return ($maxdev, $controller, $controller_prefix);
6075 }
6076
6077 # bash completion helper
6078
6079 sub complete_backup_archives {
6080     my ($cmdname, $pname, $cvalue) = @_;
6081
6082     my $cfg = PVE::Storage::config();
6083
6084     my $storeid;
6085
6086     if ($cvalue =~ m/^([^:]+):/) {
6087         $storeid = $1;
6088     }
6089
6090     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6091
6092     my $res = [];
6093     foreach my $id (keys %$data) {
6094         foreach my $item (@{$data->{$id}}) {
6095             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6096             push @$res, $item->{volid} if defined($item->{volid});
6097         }
6098     }
6099
6100     return $res;
6101 }
6102
6103 my $complete_vmid_full = sub {
6104     my ($running) = @_;
6105
6106     my $idlist = vmstatus();
6107
6108     my $res = [];
6109
6110     foreach my $id (keys %$idlist) {
6111         my $d = $idlist->{$id};
6112         if (defined($running)) {
6113             next if $d->{template};
6114             next if $running && $d->{status} ne 'running';
6115             next if !$running && $d->{status} eq 'running';
6116         }
6117         push @$res, $id;
6118
6119     }
6120     return $res;
6121 };
6122
6123 sub complete_vmid {
6124     return &$complete_vmid_full();
6125 }
6126
6127 sub complete_vmid_stopped {
6128     return &$complete_vmid_full(0);
6129 }
6130
6131 sub complete_vmid_running {
6132     return &$complete_vmid_full(1);
6133 }
6134
6135 sub complete_storage {
6136
6137     my $cfg = PVE::Storage::config();
6138     my $ids = $cfg->{ids};
6139
6140     my $res = [];
6141     foreach my $sid (keys %$ids) {
6142         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6143         next if !$ids->{$sid}->{content}->{images};
6144         push @$res, $sid;
6145     }
6146
6147     return $res;
6148 }
6149
6150 1;