PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120
 121     # AMD CPUs
 122     athlon => 'AuthenticAMD',
 123     phenom  => 'AuthenticAMD',
 124     Opteron_G1  => 'AuthenticAMD',
 125     Opteron_G2  => 'AuthenticAMD',
 126     Opteron_G3  => 'AuthenticAMD',
 127     Opteron_G4  => 'AuthenticAMD',
 128     Opteron_G5  => 'AuthenticAMD',
 129
 130     # generic types, use vendor from host node
 131     host => 'default',
 132     kvm32 => 'default',
 133     kvm64 => 'default',
 134     qemu32 => 'default',
 135     qemu64 => 'default',
 136 };
 137
 138 my $cpu_fmt = {
 139     cputype => {
 140         description => "Emulated CPU type.",
 141         type => 'string',
 142         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 143         format_description => 'cputype',
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 218         minimum => 0,
 219         maximum => 500000,
 220         default => 1024,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012
 285 l24;; Linux 2.4 Kernel
 286 l26;; Linux 2.6/3.X Kernel
 287 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 288 EODESC
 289     },
 290     boot => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 294         pattern => '[acdn]{1,4}',
 295         default => 'cdn',
 296     },
 297     bootdisk => {
 298         optional => 1,
 299         type => 'string', format => 'pve-qm-bootdisk',
 300         description => "Enable booting from specified disk.",
 301         pattern => '(ide|sata|scsi|virtio)\d+',
 302     },
 303     smp => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPUs. Please use option -sockets instead.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     sockets => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPU sockets.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     cores => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of cores per socket.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     numa => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable NUMA.",
 328         default => 0,
 329     },
 330     hugepages => {
 331         optional => 1,
 332         type => 'string',
 333         description => "Enable/disable hugepages memory.",
 334         enum => [qw(any 2 1024)],
 335     },
 336     vcpus => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Number of hotplugged vcpus.",
 340         minimum => 1,
 341         default => 0,
 342     },
 343     acpi => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable ACPI.",
 347         default => 1,
 348     },
 349     agent => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable Qemu GuestAgent.",
 353         default => 0,
 354     },
 355     kvm => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable KVM hardware virtualization.",
 359         default => 1,
 360     },
 361     tdf => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable time drift fix.",
 365         default => 0,
 366     },
 367     localtime => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 371     },
 372     freeze => {
 373         optional => 1,
 374         type => 'boolean',
 375         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 376     },
 377     vga => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Select the VGA type.",
 381         verbose_description => "Select the VGA type. If you want to use high resolution" .
 382             " modes (>= 1280x1024x16) then you should use the options " .
 383             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 384             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 385             "display sever. For win* OS you can select how many independent " .
 386             "displays you want, Linux guests can add displays them self. " .
 387             "You can also run without any graphic card, using a serial device" .
 388             " as terminal.",
 389         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 390     },
 391     watchdog => {
 392         optional => 1,
 393         type => 'string', format => 'pve-qm-watchdog',
 394         description => "Create a virtual hardware watchdog device.",
 395         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 396             " (by a guest action), the watchdog must be periodically polled " .
 397             "by an agent inside the guest or else the watchdog will reset " .
 398             "the guest (or execute the respective action specified)",
 399     },
 400     startdate => {
 401         optional => 1,
 402         type => 'string',
 403         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 404         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 405         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 406         default => 'now',
 407     },
 408     startup =>  get_standard_option('pve-startup-order'),
 409     template => {
 410         optional => 1,
 411         type => 'boolean',
 412         description => "Enable/disable Template.",
 413         default => 0,
 414     },
 415     args => {
 416         optional => 1,
 417         type => 'string',
 418         description => "Arbitrary arguments passed to kvm.",
 419         verbose_description => <<EODESCR,
 420 Arbitrary arguments passed to kvm, for example:
 421
 422 args: -no-reboot -no-hpet
 423
 424 NOTE: this option is for experts only.
 425 EODESCR
 426     },
 427     tablet => {
 428         optional => 1,
 429         type => 'boolean',
 430         default => 1,
 431         description => "Enable/disable the USB tablet device.",
 432         verbose_description => "Enable/disable the USB tablet device. This device is " .
 433             "usually needed to allow absolute mouse positioning with VNC. " .
 434             "Else the mouse runs out of sync with normal VNC clients. " .
 435             "If you're running lots of console-only guests on one host, " .
 436             "you may consider disabling this to save some context switches. " .
 437             "This is turned off by default if you use spice (-vga=qxl).",
 438     },
 439     migrate_speed => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 443         minimum => 0,
 444         default => 0,
 445     },
 446     migrate_downtime => {
 447         optional => 1,
 448         type => 'number',
 449         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 450         minimum => 0,
 451         default => 0.1,
 452     },
 453     cdrom => {
 454         optional => 1,
 455         type => 'string', format => 'pve-qm-ide',
 456         typetext => 'volume',
 457         description => "This is an alias for option -ide2",
 458     },
 459     cpu => {
 460         optional => 1,
 461         description => "Emulated CPU type.",
 462         type => 'string',
 463         format => $cpu_fmt,
 464     },
 465     parent => get_standard_option('pve-snapshot-name', {
 466         optional => 1,
 467         description => "Parent snapshot name. This is used internally, and should not be modified.",
 468     }),
 469     snaptime => {
 470         optional => 1,
 471         description => "Timestamp for snapshots.",
 472         type => 'integer',
 473         minimum => 0,
 474     },
 475     vmstate => {
 476         optional => 1,
 477         type => 'string', format => 'pve-volume-id',
 478         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 479     },
 480     machine => {
 481         description => "Specific the Qemu machine type.",
 482         type => 'string',
 483         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 484         maxLength => 40,
 485         optional => 1,
 486     },
 487     smbios1 => {
 488         description => "Specify SMBIOS type 1 fields.",
 489         type => 'string', format => 'pve-qm-smbios1',
 490         maxLength => 256,
 491         optional => 1,
 492     },
 493     protection => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 497         default => 0,
 498     },
 499     bios => {
 500         optional => 1,
 501         type => 'string',
 502         enum => [ qw(seabios ovmf) ],
 503         description => "Select BIOS implementation.",
 504         default => 'seabios',
 505     },
 506 };
 507
 508 # what about other qemu settings ?
 509 #cpu => 'string',
 510 #machine => 'string',
 511 #fda => 'file',
 512 #fdb => 'file',
 513 #mtdblock => 'file',
 514 #sd => 'file',
 515 #pflash => 'file',
 516 #snapshot => 'bool',
 517 #bootp => 'file',
 518 ##tftp => 'dir',
 519 ##smb => 'dir',
 520 #kernel => 'file',
 521 #append => 'string',
 522 #initrd => 'file',
 523 ##soundhw => 'string',
 524
 525 while (my ($k, $v) = each %$confdesc) {
 526     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 527 }
 528
 529 my $MAX_IDE_DISKS = 4;
 530 my $MAX_SCSI_DISKS = 14;
 531 my $MAX_VIRTIO_DISKS = 16;
 532 my $MAX_SATA_DISKS = 6;
 533 my $MAX_USB_DEVICES = 5;
 534 my $MAX_NETS = 32;
 535 my $MAX_UNUSED_DISKS = 8;
 536 my $MAX_HOSTPCI_DEVICES = 4;
 537 my $MAX_SERIAL_PORTS = 4;
 538 my $MAX_PARALLEL_PORTS = 3;
 539 my $MAX_NUMA = 8;
 540
 541 my $numa_fmt = {
 542     cpus => {
 543         type => "string",
 544         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 545         description => "CPUs accessing this NUMA node.",
 546         format_description => "id[-id];...",
 547     },
 548     memory => {
 549         type => "number",
 550         description => "Amount of memory this NUMA node provides.",
 551         optional => 1,
 552     },
 553     hostnodes => {
 554         type => "string",
 555         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 556         description => "Host NUMA nodes to use.",
 557         format_description => "id[-id];...",
 558         optional => 1,
 559     },
 560     policy => {
 561         type => 'string',
 562         enum => [qw(preferred bind interleave)],
 563         description => "NUMA allocation policy.",
 564         optional => 1,
 565     },
 566 };
 567 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 568 my $numadesc = {
 569     optional => 1,
 570     type => 'string', format => $numa_fmt,
 571     description => "NUMA topology.",
 572 };
 573 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 574
 575 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 576     $confdesc->{"numa$i"} = $numadesc;
 577 }
 578
 579 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 580                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 581                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 582 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 583
 584 my $net_fmt_bridge_descr = <<__EOD__;
 585 Bridge to attach the network device to. The Proxmox VE standard bridge
 586 is called 'vmbr0'.
 587
 588 If you do not specify a bridge, we create a kvm user (NATed) network
 589 device, which provides DHCP and DNS services. The following addresses
 590 are used:
 591
 592  10.0.2.2   Gateway
 593  10.0.2.3   DNS Server
 594  10.0.2.4   SMB Server
 595
 596 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 597 __EOD__
 598
 599 my $net_fmt = {
 600     macaddr => {
 601         type => 'string',
 602         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 603         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 604         format_description => "XX:XX:XX:XX:XX:XX",
 605         optional => 1,
 606     },
 607     model => {
 608         type => 'string',
 609         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 610         format_description => 'model',
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => {
 738         type => 'string',
 739         format_description => 'image format',
 740         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 741         description => "The drive's backing file's data format.",
 742         optional => 1,
 743     },
 744     size => {
 745         type => 'string',
 746         format => 'disk-size',
 747         format_description => 'DiskSize',
 748         description => "Disk size. This is purely informational and has no effect.",
 749         optional => 1,
 750     },
 751     backup => {
 752         type => 'boolean',
 753         description => "Whether the drive should be included when making backups.",
 754         optional => 1,
 755     },
 756     werror => {
 757         type => 'string',
 758         enum => [qw(enospc ignore report stop)],
 759         description => 'Write error action.',
 760         optional => 1,
 761     },
 762     aio => {
 763         type => 'string',
 764         enum => [qw(native threads)],
 765         description => 'AIO type to use.',
 766         optional => 1,
 767     },
 768     discard => {
 769         type => 'string',
 770         enum => [qw(ignore on)],
 771         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 772         optional => 1,
 773     },
 774     detect_zeroes => {
 775         type => 'boolean',
 776         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 777         optional => 1,
 778     },
 779     serial => {
 780         type => 'string',
 781         format => 'urlencoded',
 782         format_description => 'serial',
 783         maxLength => 20*3, # *3 since it's %xx url enoded
 784         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 785         optional => 1,
 786     }
 787 );
 788
 789 my %rerror_fmt = (
 790     rerror => {
 791         type => 'string',
 792         enum => [qw(ignore report stop)],
 793         description => 'Read error action.',
 794         optional => 1,
 795     },
 796 );
 797
 798 my %iothread_fmt = ( iothread => {
 799         type => 'boolean',
 800         description => "Whether to use iothreads for this drive",
 801         optional => 1,
 802 });
 803
 804 my %model_fmt = (
 805     model => {
 806         type => 'string',
 807         format => 'urlencoded',
 808         format_description => 'model',
 809         maxLength => 40*3, # *3 since it's %xx url enoded
 810         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 811         optional => 1,
 812     },
 813 );
 814
 815 my %queues_fmt = (
 816     queues => {
 817         type => 'integer',
 818         description => "Number of queues.",
 819         minimum => 2,
 820         optional => 1
 821     }
 822 );
 823
 824 my $add_throttle_desc = sub {
 825     my ($key, $type, $what, $unit, $longunit) = @_;
 826     $drivedesc_base{$key} = {
 827         type => $type,
 828         format_description => $unit,
 829         description => "Maximum $what speed in $longunit per second.",
 830         optional => 1,
 831     };
 832 };
 833 # throughput: (leaky bucket)
 834 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 835 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 836 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 837 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 838 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 839 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 840 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 841 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 842 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 843
 844 # pools: (pool of IO before throttling starts taking effect)
 845 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 846 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 847 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 848 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 849 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 850 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 851
 852 my $ide_fmt = {
 853     %drivedesc_base,
 854     %rerror_fmt,
 855     %model_fmt,
 856 };
 857 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 858
 859 my $idedesc = {
 860     optional => 1,
 861     type => 'string', format => $ide_fmt,
 862     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 865
 866 my $scsi_fmt = {
 867     %drivedesc_base,
 868     %iothread_fmt,
 869     %queues_fmt,
 870 };
 871 my $scsidesc = {
 872     optional => 1,
 873     type => 'string', format => $scsi_fmt,
 874     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 875 };
 876 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 877
 878 my $sata_fmt = {
 879     %drivedesc_base,
 880     %rerror_fmt,
 881 };
 882 my $satadesc = {
 883     optional => 1,
 884     type => 'string', format => $sata_fmt,
 885     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 886 };
 887 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 888
 889 my $virtio_fmt = {
 890     %drivedesc_base,
 891     %iothread_fmt,
 892     %rerror_fmt,
 893 };
 894 my $virtiodesc = {
 895     optional => 1,
 896     type => 'string', format => $virtio_fmt,
 897     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 898 };
 899 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 900
 901 my $alldrive_fmt = {
 902     %drivedesc_base,
 903     %rerror_fmt,
 904     %iothread_fmt,
 905     %model_fmt,
 906     %queues_fmt,
 907 };
 908
 909 my $efidisk_fmt = {
 910     volume => { alias => 'file' },
 911     file => {
 912         type => 'string',
 913         format => 'pve-volume-id-or-qm-path',
 914         default_key => 1,
 915         format_description => 'volume',
 916         description => "The drive's backing volume.",
 917     },
 918     format => {
 919         type => 'string',
 920         format_description => 'image format',
 921         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 922         description => "The drive's backing file's data format.",
 923         optional => 1,
 924     },
 925     size => {
 926         type => 'string',
 927         format => 'disk-size',
 928         format_description => 'DiskSize',
 929         description => "Disk size. This is purely informational and has no effect.",
 930         optional => 1,
 931     },
 932 };
 933
 934 my $efidisk_desc = {
 935     optional => 1,
 936     type => 'string', format => $efidisk_fmt,
 937     description => "Configure a Disk for storing EFI vars",
 938 };
 939
 940 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 941
 942 my $usb_fmt = {
 943     host => {
 944         default_key => 1,
 945         type => 'string', format => 'pve-qm-usb-device',
 946         format_description => 'HOSTUSBDEVICE|spice',
 947         description => <<EODESCR,
 948 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 949
 950  'bus-port(.port)*' (decimal numbers) or
 951  'vendor_id:product_id' (hexadeciaml numbers) or
 952  'spice'
 953
 954 You can use the 'lsusb -t' command to list existing usb devices.
 955
 956 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 957
 958 The value 'spice' can be used to add a usb redirection devices for spice.
 959 EODESCR
 960     },
 961     usb3 => {
 962         optional => 1,
 963         type => 'boolean',
 964         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 965         default => 0,
 966     },
 967 };
 968
 969 my $usbdesc = {
 970     optional => 1,
 971     type => 'string', format => $usb_fmt,
 972     description => "Configure an USB device (n is 0 to 4).",
 973 };
 974 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 975
 976 # NOTE: the match-groups of this regex are used in parse_hostpci
 977 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 978 my $hostpci_fmt = {
 979     host => {
 980         default_key => 1,
 981         type => 'string',
 982         pattern => qr/$PCIRE(;$PCIRE)*/,
 983         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 984         description => <<EODESCR,
 985 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 986 of PCI virtual functions of the host. HOSTPCIID syntax is:
 987
 988 'bus:dev.func' (hexadecimal numbers)
 989
 990 You can us the 'lspci' command to list existing PCI devices.
 991 EODESCR
 992     },
 993     rombar => {
 994         type => 'boolean',
 995         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 996         optional => 1,
 997         default => 1,
 998     },
 999     pcie => {
1000         type => 'boolean',
1001         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1002         optional => 1,
1003         default => 0,
1004     },
1005     'x-vga' => {
1006         type => 'boolean',
1007         description =>  "Enable vfio-vga device support.",
1008         optional => 1,
1009         default => 0,
1010     },
1011 };
1012 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1013
1014 my $hostpcidesc = {
1015         optional => 1,
1016         type => 'string', format => 'pve-qm-hostpci',
1017         description => "Map host PCI devices into guest.",
1018         verbose_description =>  <<EODESCR,
1019 Map host PCI devices into guest.
1020
1021 NOTE: This option allows direct access to host hardware. So it is no longer
1022 possible to migrate such machines - use with special care.
1023
1024 CAUTION: Experimental! User reported problems with this option.
1025 EODESCR
1026 };
1027 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1028
1029 my $serialdesc = {
1030         optional => 1,
1031         type => 'string',
1032         pattern => '(/dev/.+|socket)',
1033         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1034         verbose_description =>  <<EODESCR,
1035 Create a serial device inside the VM (n is 0 to 3), and pass through a
1036 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1037 host side (use 'qm terminal' to open a terminal connection).
1038
1039 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1040
1041 CAUTION: Experimental! User reported problems with this option.
1042 EODESCR
1043 };
1044
1045 my $paralleldesc= {
1046         optional => 1,
1047         type => 'string',
1048         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1049         description =>  "Map host parallel devices (n is 0 to 2).",
1050         verbose_description =>  <<EODESCR,
1051 Map host parallel devices (n is 0 to 2).
1052
1053 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1054
1055 CAUTION: Experimental! User reported problems with this option.
1056 EODESCR
1057 };
1058
1059 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1060     $confdesc->{"parallel$i"} = $paralleldesc;
1061 }
1062
1063 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1064     $confdesc->{"serial$i"} = $serialdesc;
1065 }
1066
1067 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1068     $confdesc->{"hostpci$i"} = $hostpcidesc;
1069 }
1070
1071 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1072     $drivename_hash->{"ide$i"} = 1;
1073     $confdesc->{"ide$i"} = $idedesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1077     $drivename_hash->{"sata$i"} = 1;
1078     $confdesc->{"sata$i"} = $satadesc;
1079 }
1080
1081 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1082     $drivename_hash->{"scsi$i"} = 1;
1083     $confdesc->{"scsi$i"} = $scsidesc ;
1084 }
1085
1086 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1087     $drivename_hash->{"virtio$i"} = 1;
1088     $confdesc->{"virtio$i"} = $virtiodesc;
1089 }
1090
1091 $drivename_hash->{efidisk0} = 1;
1092 $confdesc->{efidisk0} = $efidisk_desc;
1093
1094 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1095     $confdesc->{"usb$i"} = $usbdesc;
1096 }
1097
1098 my $unuseddesc = {
1099     optional => 1,
1100     type => 'string', format => 'pve-volume-id',
1101     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1102 };
1103
1104 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1105     $confdesc->{"unused$i"} = $unuseddesc;
1106 }
1107
1108 my $kvm_api_version = 0;
1109
1110 sub kvm_version {
1111
1112     return $kvm_api_version if $kvm_api_version;
1113
1114     my $fh = IO::File->new("</dev/kvm") ||
1115         return 0;
1116
1117     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1118         $kvm_api_version = $v;
1119     }
1120
1121     $fh->close();
1122
1123     return  $kvm_api_version;
1124 }
1125
1126 my $kvm_user_version;
1127
1128 sub kvm_user_version {
1129
1130     return $kvm_user_version if $kvm_user_version;
1131
1132     $kvm_user_version = 'unknown';
1133
1134     my $code = sub {
1135         my $line = shift;
1136         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1137             $kvm_user_version = $2;
1138         }
1139     };
1140
1141     eval { run_command("kvm -version", outfunc => $code); };
1142     warn $@ if $@;
1143
1144     return $kvm_user_version;
1145
1146 }
1147
1148 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1149
1150 sub valid_drive_names {
1151     # order is important - used to autoselect boot disk
1152     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1153             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1154             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1155             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1156             'efidisk0');
1157 }
1158
1159 sub is_valid_drivename {
1160     my $dev = shift;
1161
1162     return defined($drivename_hash->{$dev});
1163 }
1164
1165 sub option_exists {
1166     my $key = shift;
1167     return defined($confdesc->{$key});
1168 }
1169
1170 sub nic_models {
1171     return $nic_model_list;
1172 }
1173
1174 sub os_list_description {
1175
1176     return {
1177         other => 'Other',
1178         wxp => 'Windows XP',
1179         w2k => 'Windows 2000',
1180         w2k3 =>, 'Windows 2003',
1181         w2k8 => 'Windows 2008',
1182         wvista => 'Windows Vista',
1183         win7 => 'Windows 7',
1184         win8 => 'Windows 8/2012',
1185         l24 => 'Linux 2.4',
1186         l26 => 'Linux 2.6',
1187     };
1188 }
1189
1190 my $cdrom_path;
1191
1192 sub get_cdrom_path {
1193
1194     return  $cdrom_path if $cdrom_path;
1195
1196     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1197     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1198     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1199 }
1200
1201 sub get_iso_path {
1202     my ($storecfg, $vmid, $cdrom) = @_;
1203
1204     if ($cdrom eq 'cdrom') {
1205         return get_cdrom_path();
1206     } elsif ($cdrom eq 'none') {
1207         return '';
1208     } elsif ($cdrom =~ m|^/|) {
1209         return $cdrom;
1210     } else {
1211         return PVE::Storage::path($storecfg, $cdrom);
1212     }
1213 }
1214
1215 # try to convert old style file names to volume IDs
1216 sub filename_to_volume_id {
1217     my ($vmid, $file, $media) = @_;
1218
1219     if (!($file eq 'none' || $file eq 'cdrom' ||
1220           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1221
1222         return undef if $file =~ m|/|;
1223
1224         if ($media && $media eq 'cdrom') {
1225             $file = "local:iso/$file";
1226         } else {
1227             $file = "local:$vmid/$file";
1228         }
1229     }
1230
1231     return $file;
1232 }
1233
1234 sub verify_media_type {
1235     my ($opt, $vtype, $media) = @_;
1236
1237     return if !$media;
1238
1239     my $etype;
1240     if ($media eq 'disk') {
1241         $etype = 'images';
1242     } elsif ($media eq 'cdrom') {
1243         $etype = 'iso';
1244     } else {
1245         die "internal error";
1246     }
1247
1248     return if ($vtype eq $etype);
1249
1250     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1251 }
1252
1253 sub cleanup_drive_path {
1254     my ($opt, $storecfg, $drive) = @_;
1255
1256     # try to convert filesystem paths to volume IDs
1257
1258     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1259         ($drive->{file} !~ m|^/dev/.+|) &&
1260         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1261         ($drive->{file} !~ m/^\d+$/)) {
1262         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1263         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1264         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1265         verify_media_type($opt, $vtype, $drive->{media});
1266         $drive->{file} = $volid;
1267     }
1268
1269     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1270 }
1271
1272 sub parse_hotplug_features {
1273     my ($data) = @_;
1274
1275     my $res = {};
1276
1277     return $res if $data eq '0';
1278
1279     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1280
1281     foreach my $feature (PVE::Tools::split_list($data)) {
1282         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1283             $res->{$1} = 1;
1284         } else {
1285             die "invalid hotplug feature '$feature'\n";
1286         }
1287     }
1288     return $res;
1289 }
1290
1291 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1292 sub pve_verify_hotplug_features {
1293     my ($value, $noerr) = @_;
1294
1295     return $value if parse_hotplug_features($value);
1296
1297     return undef if $noerr;
1298
1299     die "unable to parse hotplug option\n";
1300 }
1301
1302 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1303 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1304 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1305 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1306 #        [,iothread=on][,serial=serial][,model=model]
1307
1308 sub parse_drive {
1309     my ($key, $data) = @_;
1310
1311     my ($interface, $index);
1312
1313     if ($key =~ m/^([^\d]+)(\d+)$/) {
1314         $interface = $1;
1315         $index = $2;
1316     } else {
1317         return undef;
1318     }
1319
1320     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1321                                      : $confdesc->{$key}->{format};
1322     if (!$desc) {
1323         warn "invalid drive key: $key\n";
1324         return undef;
1325     }
1326     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1327     return undef if !$res;
1328     $res->{interface} = $interface;
1329     $res->{index} = $index;
1330
1331     my $error = 0;
1332     foreach my $opt (qw(bps bps_rd bps_wr)) {
1333         if (my $bps = defined(delete $res->{$opt})) {
1334             if (defined($res->{"m$opt"})) {
1335                 warn "both $opt and m$opt specified\n";
1336                 ++$error;
1337                 next;
1338             }
1339             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1340         }
1341     }
1342     return undef if $error;
1343
1344     return undef if $res->{mbps_rd} && $res->{mbps};
1345     return undef if $res->{mbps_wr} && $res->{mbps};
1346     return undef if $res->{iops_rd} && $res->{iops};
1347     return undef if $res->{iops_wr} && $res->{iops};
1348
1349     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1350         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1351         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1352         return undef if $res->{interface} eq 'virtio';
1353     }
1354
1355     if (my $size = $res->{size}) {
1356         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1357     }
1358
1359     return $res;
1360 }
1361
1362 sub print_drive {
1363     my ($vmid, $drive) = @_;
1364     my $data = { %$drive };
1365     delete $data->{$_} for qw(index interface);
1366     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1367 }
1368
1369 sub scsi_inquiry {
1370     my($fh, $noerr) = @_;
1371
1372     my $SG_IO = 0x2285;
1373     my $SG_GET_VERSION_NUM = 0x2282;
1374
1375     my $versionbuf = "\x00" x 8;
1376     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1377     if (!$ret) {
1378         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1379         return undef;
1380     }
1381     my $version = unpack("I", $versionbuf);
1382     if ($version < 30000) {
1383         die "scsi generic interface too old\n"  if !$noerr;
1384         return undef;
1385     }
1386
1387     my $buf = "\x00" x 36;
1388     my $sensebuf = "\x00" x 8;
1389     my $cmd = pack("C x3 C x1", 0x12, 36);
1390
1391     # see /usr/include/scsi/sg.h
1392     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1393
1394     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1395                       length($sensebuf), 0, length($buf), $buf,
1396                       $cmd, $sensebuf, 6000);
1397
1398     $ret = ioctl($fh, $SG_IO, $packet);
1399     if (!$ret) {
1400         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1401         return undef;
1402     }
1403
1404     my @res = unpack($sg_io_hdr_t, $packet);
1405     if ($res[17] || $res[18]) {
1406         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1407         return undef;
1408     }
1409
1410     my $res = {};
1411     (my $byte0, my $byte1, $res->{vendor},
1412      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1413
1414     $res->{removable} = $byte1 & 128 ? 1 : 0;
1415     $res->{type} = $byte0 & 31;
1416
1417     return $res;
1418 }
1419
1420 sub path_is_scsi {
1421     my ($path) = @_;
1422
1423     my $fh = IO::File->new("+<$path") || return undef;
1424     my $res = scsi_inquiry($fh, 1);
1425     close($fh);
1426
1427     return $res;
1428 }
1429
1430 sub machine_type_is_q35 {
1431     my ($conf) = @_;
1432
1433     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1434 }
1435
1436 sub print_tabletdevice_full {
1437     my ($conf) = @_;
1438
1439     my $q35 = machine_type_is_q35($conf);
1440
1441     # we use uhci for old VMs because tablet driver was buggy in older qemu
1442     my $usbbus = $q35 ? "ehci" : "uhci";
1443
1444     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1445 }
1446
1447 sub print_drivedevice_full {
1448     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1449
1450     my $device = '';
1451     my $maxdev = 0;
1452
1453     if ($drive->{interface} eq 'virtio') {
1454         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1455         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1456         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1457     } elsif ($drive->{interface} eq 'scsi') {
1458
1459         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1460         my $unit = $drive->{index} % $maxdev;
1461         my $devicetype = 'hd';
1462         my $path = '';
1463         if (drive_is_cdrom($drive)) {
1464             $devicetype = 'cd';
1465         } else {
1466             if ($drive->{file} =~ m|^/|) {
1467                 $path = $drive->{file};
1468                 if (my $info = path_is_scsi($path)) {
1469                     if ($info->{type} == 0) {
1470                         $devicetype = 'block';
1471                     } elsif ($info->{type} == 1) { # tape
1472                         $devicetype = 'generic';
1473                     }
1474                 }
1475             } else {
1476                  $path = PVE::Storage::path($storecfg, $drive->{file});
1477             }
1478
1479             if($path =~ m/^iscsi\:\/\//){
1480                 $devicetype = 'generic';
1481             }
1482         }
1483
1484         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1485             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1486         } else {
1487             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1488         }
1489
1490     } elsif ($drive->{interface} eq 'ide'){
1491         $maxdev = 2;
1492         my $controller = int($drive->{index} / $maxdev);
1493         my $unit = $drive->{index} % $maxdev;
1494         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1495
1496         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1497         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1498             $model = URI::Escape::uri_unescape($model);
1499             $device .= ",model=$model";
1500         }
1501     } elsif ($drive->{interface} eq 'sata'){
1502         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1503         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1504         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1505     } elsif ($drive->{interface} eq 'usb') {
1506         die "implement me";
1507         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1508     } else {
1509         die "unsupported interface type";
1510     }
1511
1512     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1513
1514     return $device;
1515 }
1516
1517 sub get_initiator_name {
1518     my $initiator;
1519
1520     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1521     while (defined(my $line = <$fh>)) {
1522         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1523         $initiator = $1;
1524         last;
1525     }
1526     $fh->close();
1527
1528     return $initiator;
1529 }
1530
1531 sub print_drive_full {
1532     my ($storecfg, $vmid, $drive) = @_;
1533
1534     my $path;
1535     my $volid = $drive->{file};
1536     my $format;
1537
1538     if (drive_is_cdrom($drive)) {
1539         $path = get_iso_path($storecfg, $vmid, $volid);
1540     } else {
1541         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1542         if ($storeid) {
1543             $path = PVE::Storage::path($storecfg, $volid);
1544             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1545             $format = qemu_img_format($scfg, $volname);
1546         } else {
1547             $path = $volid;
1548             $format = "raw";
1549         }
1550    }
1551
1552     my $opts = '';
1553     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1554     foreach my $o (@qemu_drive_options) {
1555         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1556     }
1557     if (my $serial = $drive->{serial}) {
1558         $serial = URI::Escape::uri_unescape($serial);
1559         $opts .= ",serial=$serial";
1560     }
1561
1562     $opts .= ",format=$format" if $format && !$drive->{format};
1563
1564     foreach my $o (qw(bps bps_rd bps_wr)) {
1565         my $v = $drive->{"m$o"};
1566         $opts .= ",$o=" . int($v*1024*1024) if $v;
1567     }
1568
1569     my $cache_direct = 0;
1570
1571     if (my $cache = $drive->{cache}) {
1572         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1573     } elsif (!drive_is_cdrom($drive)) {
1574         $opts .= ",cache=none";
1575         $cache_direct = 1;
1576     }
1577
1578     # aio native works only with O_DIRECT
1579     if (!$drive->{aio}) {
1580         if($cache_direct) {
1581             $opts .= ",aio=native";
1582         } else {
1583             $opts .= ",aio=threads";
1584         }
1585     }
1586
1587     if (!drive_is_cdrom($drive)) {
1588         my $detectzeroes;
1589         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1590             $detectzeroes = 'off';
1591         } elsif ($drive->{discard}) {
1592             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1593         } else {
1594             # This used to be our default with discard not being specified:
1595             $detectzeroes = 'on';
1596         }
1597         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1598     }
1599
1600     my $pathinfo = $path ? "file=$path," : '';
1601
1602     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1603 }
1604
1605 sub print_netdevice_full {
1606     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1607
1608     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1609
1610     my $device = $net->{model};
1611     if ($net->{model} eq 'virtio') {
1612          $device = 'virtio-net-pci';
1613      };
1614
1615     my $pciaddr = print_pci_addr("$netid", $bridges);
1616     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1617     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1618         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1619         my $vectors = $net->{queues} * 2 + 2;
1620         $tmpstr .= ",vectors=$vectors,mq=on";
1621     }
1622     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1623
1624     if ($use_old_bios_files) {
1625         my $romfile;
1626         if ($device eq 'virtio-net-pci') {
1627             $romfile = 'pxe-virtio.rom';
1628         } elsif ($device eq 'e1000') {
1629             $romfile = 'pxe-e1000.rom';
1630         } elsif ($device eq 'ne2k') {
1631             $romfile = 'pxe-ne2k_pci.rom';
1632         } elsif ($device eq 'pcnet') {
1633             $romfile = 'pxe-pcnet.rom';
1634         } elsif ($device eq 'rtl8139') {
1635             $romfile = 'pxe-rtl8139.rom';
1636         }
1637         $tmpstr .= ",romfile=$romfile" if $romfile;
1638     }
1639
1640     return $tmpstr;
1641 }
1642
1643 sub print_netdev_full {
1644     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1645
1646     my $i = '';
1647     if ($netid =~ m/^net(\d+)$/) {
1648         $i = int($1);
1649     }
1650
1651     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1652
1653     my $ifname = "tap${vmid}i$i";
1654
1655     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1656     die "interface name '$ifname' is too long (max 15 character)\n"
1657         if length($ifname) >= 16;
1658
1659     my $vhostparam = '';
1660     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1661
1662     my $vmname = $conf->{name} || "vm$vmid";
1663
1664     my $netdev = "";
1665     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1666
1667     if ($net->{bridge}) {
1668         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1669     } else {
1670         $netdev = "type=user,id=$netid,hostname=$vmname";
1671     }
1672
1673     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1674
1675     return $netdev;
1676 }
1677
1678
1679 sub print_cpu_device {
1680     my ($conf, $id) = @_;
1681
1682     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1683     my $cpu = $nokvm ? "qemu64" : "kvm64";
1684     if (my $cputype = $conf->{cpu}) {
1685         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1686             or die "Cannot parse cpu description: $cputype\n";
1687         $cpu = $cpuconf->{cputype};
1688     }
1689
1690     my $sockets = 1;
1691     $sockets = $conf->{sockets} if  $conf->{sockets};
1692     my $cores = $conf->{cores} || 1;
1693
1694     my $current_core = ($id - 1) % $cores;
1695     my $current_socket = int(($id - $current_core)/$cores);
1696
1697     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1698 }
1699
1700 sub drive_is_cdrom {
1701     my ($drive) = @_;
1702
1703     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1704
1705 }
1706
1707 sub parse_number_sets {
1708     my ($set) = @_;
1709     my $res = [];
1710     foreach my $part (split(/;/, $set)) {
1711         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1712             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1713             push @$res, [ $1, $2 ];
1714         } else {
1715             die "invalid range: $part\n";
1716         }
1717     }
1718     return $res;
1719 }
1720
1721 sub parse_numa {
1722     my ($data) = @_;
1723
1724     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1725     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1726     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1727     return $res;
1728 }
1729
1730 sub parse_hostpci {
1731     my ($value) = @_;
1732
1733     return undef if !$value;
1734
1735     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1736
1737     my @idlist = split(/;/, $res->{host});
1738     delete $res->{host};
1739     foreach my $id (@idlist) {
1740         if ($id =~ /^$PCIRE$/) {
1741             if (defined($2)) {
1742                 push @{$res->{pciid}}, { id => $1, function => $2 };
1743             } else {
1744                 my $pcidevices = lspci($1);
1745                 $res->{pciid} = $pcidevices->{$1};
1746             }
1747         } else {
1748             # should have been caught by parse_property_string already
1749             die "failed to parse PCI id: $id\n";
1750         }
1751     }
1752     return $res;
1753 }
1754
1755 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1756 sub parse_net {
1757     my ($data) = @_;
1758
1759     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1760     if ($@) {
1761         warn $@;
1762         return undef;
1763     }
1764     if (!defined($res->{macaddr})) {
1765         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1766         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1767     }
1768     return $res;
1769 }
1770
1771 sub print_net {
1772     my $net = shift;
1773
1774     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1775 }
1776
1777 sub add_random_macs {
1778     my ($settings) = @_;
1779
1780     foreach my $opt (keys %$settings) {
1781         next if $opt !~ m/^net(\d+)$/;
1782         my $net = parse_net($settings->{$opt});
1783         next if !$net;
1784         $settings->{$opt} = print_net($net);
1785     }
1786 }
1787
1788 sub vm_is_volid_owner {
1789     my ($storecfg, $vmid, $volid) = @_;
1790
1791     if ($volid !~  m|^/|) {
1792         my ($path, $owner);
1793         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1794         if ($owner && ($owner == $vmid)) {
1795             return 1;
1796         }
1797     }
1798
1799     return undef;
1800 }
1801
1802 sub split_flagged_list {
1803     my $text = shift || '';
1804     $text =~ s/[,;]/ /g;
1805     $text =~ s/^\s+//;
1806     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1807 }
1808
1809 sub join_flagged_list {
1810     my ($how, $lst) = @_;
1811     join $how, map { $lst->{$_} . $_ } keys %$lst;
1812 }
1813
1814 sub vmconfig_delete_pending_option {
1815     my ($conf, $key, $force) = @_;
1816
1817     delete $conf->{pending}->{$key};
1818     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1819     $pending_delete_hash->{$key} = $force ? '!' : '';
1820     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1821 }
1822
1823 sub vmconfig_undelete_pending_option {
1824     my ($conf, $key) = @_;
1825
1826     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1827     delete $pending_delete_hash->{$key};
1828
1829     if (%$pending_delete_hash) {
1830         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1831     } else {
1832         delete $conf->{pending}->{delete};
1833     }
1834 }
1835
1836 sub vmconfig_register_unused_drive {
1837     my ($storecfg, $vmid, $conf, $drive) = @_;
1838
1839     if (!drive_is_cdrom($drive)) {
1840         my $volid = $drive->{file};
1841         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1842             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1843         }
1844     }
1845 }
1846
1847 sub vmconfig_cleanup_pending {
1848     my ($conf) = @_;
1849
1850     # remove pending changes when nothing changed
1851     my $changes;
1852     foreach my $opt (keys %{$conf->{pending}}) {
1853         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1854             $changes = 1;
1855             delete $conf->{pending}->{$opt};
1856         }
1857     }
1858
1859     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1860     my $pending_delete_hash = {};
1861     while (my ($opt, $force) = each %$current_delete_hash) {
1862         if (defined($conf->{$opt})) {
1863             $pending_delete_hash->{$opt} = $force;
1864         } else {
1865             $changes = 1;
1866         }
1867     }
1868
1869     if (%$pending_delete_hash) {
1870         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1871     } else {
1872         delete $conf->{pending}->{delete};
1873     }
1874
1875     return $changes;
1876 }
1877
1878 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1879 my $smbios1_fmt = {
1880     uuid => {
1881         type => 'string',
1882         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1883         format_description => 'UUID',
1884         description => "Set SMBIOS1 UUID.",
1885         optional => 1,
1886     },
1887     version => {
1888         type => 'string',
1889         pattern => '\S+',
1890         format_description => 'string',
1891         description => "Set SMBIOS1 version.",
1892         optional => 1,
1893     },
1894     serial => {
1895         type => 'string',
1896         pattern => '\S+',
1897         format_description => 'string',
1898         description => "Set SMBIOS1 serial number.",
1899         optional => 1,
1900     },
1901     manufacturer => {
1902         type => 'string',
1903         pattern => '\S+',
1904         format_description => 'string',
1905         description => "Set SMBIOS1 manufacturer.",
1906         optional => 1,
1907     },
1908     product => {
1909         type => 'string',
1910         pattern => '\S+',
1911         format_description => 'string',
1912         description => "Set SMBIOS1 product ID.",
1913         optional => 1,
1914     },
1915     sku => {
1916         type => 'string',
1917         pattern => '\S+',
1918         format_description => 'string',
1919         description => "Set SMBIOS1 SKU string.",
1920         optional => 1,
1921     },
1922     family => {
1923         type => 'string',
1924         pattern => '\S+',
1925         format_description => 'string',
1926         description => "Set SMBIOS1 family string.",
1927         optional => 1,
1928     },
1929 };
1930
1931 sub parse_smbios1 {
1932     my ($data) = @_;
1933
1934     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1935     warn $@ if $@;
1936     return $res;
1937 }
1938
1939 sub print_smbios1 {
1940     my ($smbios1) = @_;
1941     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1942 }
1943
1944 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1945
1946 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1947 sub verify_bootdisk {
1948     my ($value, $noerr) = @_;
1949
1950     return $value if is_valid_drivename($value);
1951
1952     return undef if $noerr;
1953
1954     die "invalid boot disk '$value'\n";
1955 }
1956
1957 sub parse_watchdog {
1958     my ($value) = @_;
1959
1960     return undef if !$value;
1961
1962     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1963     warn $@ if $@;
1964     return $res;
1965 }
1966
1967 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1968 sub verify_usb_device {
1969     my ($value, $noerr) = @_;
1970
1971     return $value if parse_usb_device($value);
1972
1973     return undef if $noerr;
1974
1975     die "unable to parse usb device\n";
1976 }
1977
1978 # add JSON properties for create and set function
1979 sub json_config_properties {
1980     my $prop = shift;
1981
1982     foreach my $opt (keys %$confdesc) {
1983         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1984         $prop->{$opt} = $confdesc->{$opt};
1985     }
1986
1987     return $prop;
1988 }
1989
1990 sub check_type {
1991     my ($key, $value) = @_;
1992
1993     die "unknown setting '$key'\n" if !$confdesc->{$key};
1994
1995     my $type = $confdesc->{$key}->{type};
1996
1997     if (!defined($value)) {
1998         die "got undefined value\n";
1999     }
2000
2001     if ($value =~ m/[\n\r]/) {
2002         die "property contains a line feed\n";
2003     }
2004
2005     if ($type eq 'boolean') {
2006         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2007         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2008         die "type check ('boolean') failed - got '$value'\n";
2009     } elsif ($type eq 'integer') {
2010         return int($1) if $value =~ m/^(\d+)$/;
2011         die "type check ('integer') failed - got '$value'\n";
2012     } elsif ($type eq 'number') {
2013         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2014         die "type check ('number') failed - got '$value'\n";
2015     } elsif ($type eq 'string') {
2016         if (my $fmt = $confdesc->{$key}->{format}) {
2017             PVE::JSONSchema::check_format($fmt, $value);
2018             return $value;
2019         }
2020         $value =~ s/^\"(.*)\"$/$1/;
2021         return $value;
2022     } else {
2023         die "internal error"
2024     }
2025 }
2026
2027 sub check_iommu_support{
2028     #fixme : need to check IOMMU support
2029     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2030
2031     my $iommu=1;
2032     return $iommu;
2033
2034 }
2035
2036 sub touch_config {
2037     my ($vmid) = @_;
2038
2039     my $conf = PVE::QemuConfig->config_file($vmid);
2040     utime undef, undef, $conf;
2041 }
2042
2043 sub destroy_vm {
2044     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2045
2046     my $conffile = PVE::QemuConfig->config_file($vmid);
2047
2048     my $conf = PVE::QemuConfig->load_config($vmid);
2049
2050     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2051
2052     # only remove disks owned by this VM
2053     foreach_drive($conf, sub {
2054         my ($ds, $drive) = @_;
2055
2056         return if drive_is_cdrom($drive);
2057
2058         my $volid = $drive->{file};
2059
2060         return if !$volid || $volid =~ m|^/|;
2061
2062         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2063         return if !$path || !$owner || ($owner != $vmid);
2064
2065         PVE::Storage::vdisk_free($storecfg, $volid);
2066     });
2067
2068     if ($keep_empty_config) {
2069         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2070     } else {
2071         unlink $conffile;
2072     }
2073
2074     # also remove unused disk
2075     eval {
2076         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2077
2078         eval {
2079             PVE::Storage::foreach_volid($dl, sub {
2080                 my ($volid, $sid, $volname, $d) = @_;
2081                 PVE::Storage::vdisk_free($storecfg, $volid);
2082             });
2083         };
2084         warn $@ if $@;
2085
2086     };
2087     warn $@ if $@;
2088 }
2089
2090 sub parse_vm_config {
2091     my ($filename, $raw) = @_;
2092
2093     return undef if !defined($raw);
2094
2095     my $res = {
2096         digest => Digest::SHA::sha1_hex($raw),
2097         snapshots => {},
2098         pending => {},
2099     };
2100
2101     $filename =~ m|/qemu-server/(\d+)\.conf$|
2102         || die "got strange filename '$filename'";
2103
2104     my $vmid = $1;
2105
2106     my $conf = $res;
2107     my $descr;
2108     my $section = '';
2109
2110     my @lines = split(/\n/, $raw);
2111     foreach my $line (@lines) {
2112         next if $line =~ m/^\s*$/;
2113
2114         if ($line =~ m/^\[PENDING\]\s*$/i) {
2115             $section = 'pending';
2116             if (defined($descr)) {
2117                 $descr =~ s/\s+$//;
2118                 $conf->{description} = $descr;
2119             }
2120             $descr = undef;
2121             $conf = $res->{$section} = {};
2122             next;
2123
2124         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2125             $section = $1;
2126             if (defined($descr)) {
2127                 $descr =~ s/\s+$//;
2128                 $conf->{description} = $descr;
2129             }
2130             $descr = undef;
2131             $conf = $res->{snapshots}->{$section} = {};
2132             next;
2133         }
2134
2135         if ($line =~ m/^\#(.*)\s*$/) {
2136             $descr = '' if !defined($descr);
2137             $descr .= PVE::Tools::decode_text($1) . "\n";
2138             next;
2139         }
2140
2141         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2142             $descr = '' if !defined($descr);
2143             $descr .= PVE::Tools::decode_text($2);
2144         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2145             $conf->{snapstate} = $1;
2146         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2147             my $key = $1;
2148             my $value = $2;
2149             $conf->{$key} = $value;
2150         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2151             my $value = $1;
2152             if ($section eq 'pending') {
2153                 $conf->{delete} = $value; # we parse this later
2154             } else {
2155                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2156             }
2157         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2158             my $key = $1;
2159             my $value = $2;
2160             eval { $value = check_type($key, $value); };
2161             if ($@) {
2162                 warn "vm $vmid - unable to parse value of '$key' - $@";
2163             } else {
2164                 $key = 'ide2' if $key eq 'cdrom';
2165                 my $fmt = $confdesc->{$key}->{format};
2166                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2167                     my $v = parse_drive($key, $value);
2168                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2169                         $v->{file} = $volid;
2170                         $value = print_drive($vmid, $v);
2171                     } else {
2172                         warn "vm $vmid - unable to parse value of '$key'\n";
2173                         next;
2174                     }
2175                 }
2176
2177                 $conf->{$key} = $value;
2178             }
2179         }
2180     }
2181
2182     if (defined($descr)) {
2183         $descr =~ s/\s+$//;
2184         $conf->{description} = $descr;
2185     }
2186     delete $res->{snapstate}; # just to be sure
2187
2188     return $res;
2189 }
2190
2191 sub write_vm_config {
2192     my ($filename, $conf) = @_;
2193
2194     delete $conf->{snapstate}; # just to be sure
2195
2196     if ($conf->{cdrom}) {
2197         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2198         $conf->{ide2} = $conf->{cdrom};
2199         delete $conf->{cdrom};
2200     }
2201
2202     # we do not use 'smp' any longer
2203     if ($conf->{sockets}) {
2204         delete $conf->{smp};
2205     } elsif ($conf->{smp}) {
2206         $conf->{sockets} = $conf->{smp};
2207         delete $conf->{cores};
2208         delete $conf->{smp};
2209     }
2210
2211     my $used_volids = {};
2212
2213     my $cleanup_config = sub {
2214         my ($cref, $pending, $snapname) = @_;
2215
2216         foreach my $key (keys %$cref) {
2217             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2218                 $key eq 'snapstate' || $key eq 'pending';
2219             my $value = $cref->{$key};
2220             if ($key eq 'delete') {
2221                 die "propertry 'delete' is only allowed in [PENDING]\n"
2222                     if !$pending;
2223                 # fixme: check syntax?
2224                 next;
2225             }
2226             eval { $value = check_type($key, $value); };
2227             die "unable to parse value of '$key' - $@" if $@;
2228
2229             $cref->{$key} = $value;
2230
2231             if (!$snapname && is_valid_drivename($key)) {
2232                 my $drive = parse_drive($key, $value);
2233                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2234             }
2235         }
2236     };
2237
2238     &$cleanup_config($conf);
2239
2240     &$cleanup_config($conf->{pending}, 1);
2241
2242     foreach my $snapname (keys %{$conf->{snapshots}}) {
2243         die "internal error" if $snapname eq 'pending';
2244         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2245     }
2246
2247     # remove 'unusedX' settings if we re-add a volume
2248     foreach my $key (keys %$conf) {
2249         my $value = $conf->{$key};
2250         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2251             delete $conf->{$key};
2252         }
2253     }
2254
2255     my $generate_raw_config = sub {
2256         my ($conf, $pending) = @_;
2257
2258         my $raw = '';
2259
2260         # add description as comment to top of file
2261         if (defined(my $descr = $conf->{description})) {
2262             if ($descr) {
2263                 foreach my $cl (split(/\n/, $descr)) {
2264                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2265                 }
2266             } else {
2267                 $raw .= "#\n" if $pending;
2268             }
2269         }
2270
2271         foreach my $key (sort keys %$conf) {
2272             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2273             $raw .= "$key: $conf->{$key}\n";
2274         }
2275         return $raw;
2276     };
2277
2278     my $raw = &$generate_raw_config($conf);
2279
2280     if (scalar(keys %{$conf->{pending}})){
2281         $raw .= "\n[PENDING]\n";
2282         $raw .= &$generate_raw_config($conf->{pending}, 1);
2283     }
2284
2285     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2286         $raw .= "\n[$snapname]\n";
2287         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2288     }
2289
2290     return $raw;
2291 }
2292
2293 sub load_defaults {
2294
2295     my $res = {};
2296
2297     # we use static defaults from our JSON schema configuration
2298     foreach my $key (keys %$confdesc) {
2299         if (defined(my $default = $confdesc->{$key}->{default})) {
2300             $res->{$key} = $default;
2301         }
2302     }
2303
2304     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2305     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2306
2307     return $res;
2308 }
2309
2310 sub config_list {
2311     my $vmlist = PVE::Cluster::get_vmlist();
2312     my $res = {};
2313     return $res if !$vmlist || !$vmlist->{ids};
2314     my $ids = $vmlist->{ids};
2315
2316     foreach my $vmid (keys %$ids) {
2317         my $d = $ids->{$vmid};
2318         next if !$d->{node} || $d->{node} ne $nodename;
2319         next if !$d->{type} || $d->{type} ne 'qemu';
2320         $res->{$vmid}->{exists} = 1;
2321     }
2322     return $res;
2323 }
2324
2325 # test if VM uses local resources (to prevent migration)
2326 sub check_local_resources {
2327     my ($conf, $noerr) = @_;
2328
2329     my $loc_res = 0;
2330
2331     $loc_res = 1 if $conf->{hostusb}; # old syntax
2332     $loc_res = 1 if $conf->{hostpci}; # old syntax
2333
2334     foreach my $k (keys %$conf) {
2335         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2336         # sockets are safe: they will recreated be on the target side post-migrate
2337         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2338         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2339     }
2340
2341     die "VM uses local resources\n" if $loc_res && !$noerr;
2342
2343     return $loc_res;
2344 }
2345
2346 # check if used storages are available on all nodes (use by migrate)
2347 sub check_storage_availability {
2348     my ($storecfg, $conf, $node) = @_;
2349
2350     foreach_drive($conf, sub {
2351         my ($ds, $drive) = @_;
2352
2353         my $volid = $drive->{file};
2354         return if !$volid;
2355
2356         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2357         return if !$sid;
2358
2359         # check if storage is available on both nodes
2360         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2361         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2362    });
2363 }
2364
2365 # list nodes where all VM images are available (used by has_feature API)
2366 sub shared_nodes {
2367     my ($conf, $storecfg) = @_;
2368
2369     my $nodelist = PVE::Cluster::get_nodelist();
2370     my $nodehash = { map { $_ => 1 } @$nodelist };
2371     my $nodename = PVE::INotify::nodename();
2372
2373     foreach_drive($conf, sub {
2374         my ($ds, $drive) = @_;
2375
2376         my $volid = $drive->{file};
2377         return if !$volid;
2378
2379         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2380         if ($storeid) {
2381             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2382             if ($scfg->{disable}) {
2383                 $nodehash = {};
2384             } elsif (my $avail = $scfg->{nodes}) {
2385                 foreach my $node (keys %$nodehash) {
2386                     delete $nodehash->{$node} if !$avail->{$node};
2387                 }
2388             } elsif (!$scfg->{shared}) {
2389                 foreach my $node (keys %$nodehash) {
2390                     delete $nodehash->{$node} if $node ne $nodename
2391                 }
2392             }
2393         }
2394     });
2395
2396     return $nodehash
2397 }
2398
2399 sub check_cmdline {
2400     my ($pidfile, $pid) = @_;
2401
2402     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2403     if (defined($fh)) {
2404         my $line = <$fh>;
2405         $fh->close;
2406         return undef if !$line;
2407         my @param = split(/\0/, $line);
2408
2409         my $cmd = $param[0];
2410         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2411
2412         for (my $i = 0; $i < scalar (@param); $i++) {
2413             my $p = $param[$i];
2414             next if !$p;
2415             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2416                 my $p = $param[$i+1];
2417                 return 1 if $p && ($p eq $pidfile);
2418                 return undef;
2419             }
2420         }
2421     }
2422     return undef;
2423 }
2424
2425 sub check_running {
2426     my ($vmid, $nocheck, $node) = @_;
2427
2428     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2429
2430     die "unable to find configuration file for VM $vmid - no such machine\n"
2431         if !$nocheck && ! -f $filename;
2432
2433     my $pidfile = pidfile_name($vmid);
2434
2435     if (my $fd = IO::File->new("<$pidfile")) {
2436         my $st = stat($fd);
2437         my $line = <$fd>;
2438         close($fd);
2439
2440         my $mtime = $st->mtime;
2441         if ($mtime > time()) {
2442             warn "file '$filename' modified in future\n";
2443         }
2444
2445         if ($line =~ m/^(\d+)$/) {
2446             my $pid = $1;
2447             if (check_cmdline($pidfile, $pid)) {
2448                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2449                     return $pid;
2450                 }
2451             }
2452         }
2453     }
2454
2455     return undef;
2456 }
2457
2458 sub vzlist {
2459
2460     my $vzlist = config_list();
2461
2462     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2463
2464     while (defined(my $de = $fd->read)) {
2465         next if $de !~ m/^(\d+)\.pid$/;
2466         my $vmid = $1;
2467         next if !defined($vzlist->{$vmid});
2468         if (my $pid = check_running($vmid)) {
2469             $vzlist->{$vmid}->{pid} = $pid;
2470         }
2471     }
2472
2473     return $vzlist;
2474 }
2475
2476 sub disksize {
2477     my ($storecfg, $conf) = @_;
2478
2479     my $bootdisk = $conf->{bootdisk};
2480     return undef if !$bootdisk;
2481     return undef if !is_valid_drivename($bootdisk);
2482
2483     return undef if !$conf->{$bootdisk};
2484
2485     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2486     return undef if !defined($drive);
2487
2488     return undef if drive_is_cdrom($drive);
2489
2490     my $volid = $drive->{file};
2491     return undef if !$volid;
2492
2493     return $drive->{size};
2494 }
2495
2496 my $last_proc_pid_stat;
2497
2498 # get VM status information
2499 # This must be fast and should not block ($full == false)
2500 # We only query KVM using QMP if $full == true (this can be slow)
2501 sub vmstatus {
2502     my ($opt_vmid, $full) = @_;
2503
2504     my $res = {};
2505
2506     my $storecfg = PVE::Storage::config();
2507
2508     my $list = vzlist();
2509     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2510
2511     my $cpucount = $cpuinfo->{cpus} || 1;
2512
2513     foreach my $vmid (keys %$list) {
2514         next if $opt_vmid && ($vmid ne $opt_vmid);
2515
2516         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2517         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2518
2519         my $d = {};
2520         $d->{pid} = $list->{$vmid}->{pid};
2521
2522         # fixme: better status?
2523         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2524
2525         my $size = disksize($storecfg, $conf);
2526         if (defined($size)) {
2527             $d->{disk} = 0; # no info available
2528             $d->{maxdisk} = $size;
2529         } else {
2530             $d->{disk} = 0;
2531             $d->{maxdisk} = 0;
2532         }
2533
2534         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2535         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2536         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2537
2538         $d->{name} = $conf->{name} || "VM $vmid";
2539         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2540
2541         if ($conf->{balloon}) {
2542             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2543             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2544         }
2545
2546         $d->{uptime} = 0;
2547         $d->{cpu} = 0;
2548         $d->{mem} = 0;
2549
2550         $d->{netout} = 0;
2551         $d->{netin} = 0;
2552
2553         $d->{diskread} = 0;
2554         $d->{diskwrite} = 0;
2555
2556         $d->{template} = PVE::QemuConfig->is_template($conf);
2557
2558         $res->{$vmid} = $d;
2559     }
2560
2561     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2562     foreach my $dev (keys %$netdev) {
2563         next if $dev !~ m/^tap([1-9]\d*)i/;
2564         my $vmid = $1;
2565         my $d = $res->{$vmid};
2566         next if !$d;
2567
2568         $d->{netout} += $netdev->{$dev}->{receive};
2569         $d->{netin} += $netdev->{$dev}->{transmit};
2570
2571         if ($full) {
2572             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2573             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2574         }
2575
2576     }
2577
2578     my $ctime = gettimeofday;
2579
2580     foreach my $vmid (keys %$list) {
2581
2582         my $d = $res->{$vmid};
2583         my $pid = $d->{pid};
2584         next if !$pid;
2585
2586         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2587         next if !$pstat; # not running
2588
2589         my $used = $pstat->{utime} + $pstat->{stime};
2590
2591         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2592
2593         if ($pstat->{vsize}) {
2594             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2595         }
2596
2597         my $old = $last_proc_pid_stat->{$pid};
2598         if (!$old) {
2599             $last_proc_pid_stat->{$pid} = {
2600                 time => $ctime,
2601                 used => $used,
2602                 cpu => 0,
2603             };
2604             next;
2605         }
2606
2607         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2608
2609         if ($dtime > 1000) {
2610             my $dutime = $used -  $old->{used};
2611
2612             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2613             $last_proc_pid_stat->{$pid} = {
2614                 time => $ctime,
2615                 used => $used,
2616                 cpu => $d->{cpu},
2617             };
2618         } else {
2619             $d->{cpu} = $old->{cpu};
2620         }
2621     }
2622
2623     return $res if !$full;
2624
2625     my $qmpclient = PVE::QMPClient->new();
2626
2627     my $ballooncb = sub {
2628         my ($vmid, $resp) = @_;
2629
2630         my $info = $resp->{'return'};
2631         return if !$info->{max_mem};
2632
2633         my $d = $res->{$vmid};
2634
2635         # use memory assigned to VM
2636         $d->{maxmem} = $info->{max_mem};
2637         $d->{balloon} = $info->{actual};
2638
2639         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2640             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2641             $d->{freemem} = $info->{free_mem};
2642         }
2643
2644         $d->{ballooninfo} = $info;
2645     };
2646
2647     my $blockstatscb = sub {
2648         my ($vmid, $resp) = @_;
2649         my $data = $resp->{'return'} || [];
2650         my $totalrdbytes = 0;
2651         my $totalwrbytes = 0;
2652
2653         for my $blockstat (@$data) {
2654             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2655             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2656
2657             $blockstat->{device} =~ s/drive-//;
2658             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2659         }
2660         $res->{$vmid}->{diskread} = $totalrdbytes;
2661         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2662     };
2663
2664     my $statuscb = sub {
2665         my ($vmid, $resp) = @_;
2666
2667         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2668         # this fails if ballon driver is not loaded, so this must be
2669         # the last commnand (following command are aborted if this fails).
2670         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2671
2672         my $status = 'unknown';
2673         if (!defined($status = $resp->{'return'}->{status})) {
2674             warn "unable to get VM status\n";
2675             return;
2676         }
2677
2678         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2679     };
2680
2681     foreach my $vmid (keys %$list) {
2682         next if $opt_vmid && ($vmid ne $opt_vmid);
2683         next if !$res->{$vmid}->{pid}; # not running
2684         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2685     }
2686
2687     $qmpclient->queue_execute(undef, 1);
2688
2689     foreach my $vmid (keys %$list) {
2690         next if $opt_vmid && ($vmid ne $opt_vmid);
2691         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2692     }
2693
2694     return $res;
2695 }
2696
2697 sub foreach_drive {
2698     my ($conf, $func, @param) = @_;
2699
2700     foreach my $ds (valid_drive_names()) {
2701         next if !defined($conf->{$ds});
2702
2703         my $drive = parse_drive($ds, $conf->{$ds});
2704         next if !$drive;
2705
2706         &$func($ds, $drive, @param);
2707     }
2708 }
2709
2710 sub foreach_volid {
2711     my ($conf, $func, @param) = @_;
2712
2713     my $volhash = {};
2714
2715     my $test_volid = sub {
2716         my ($volid, $is_cdrom) = @_;
2717
2718         return if !$volid;
2719
2720         $volhash->{$volid} = $is_cdrom || 0;
2721     };
2722
2723     foreach_drive($conf, sub {
2724         my ($ds, $drive) = @_;
2725         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2726     });
2727
2728     foreach my $snapname (keys %{$conf->{snapshots}}) {
2729         my $snap = $conf->{snapshots}->{$snapname};
2730         &$test_volid($snap->{vmstate}, 0);
2731         foreach_drive($snap, sub {
2732             my ($ds, $drive) = @_;
2733             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2734         });
2735     }
2736
2737     foreach my $volid (keys %$volhash) {
2738         &$func($volid, $volhash->{$volid}, @param);
2739     }
2740 }
2741
2742 sub vga_conf_has_spice {
2743     my ($vga) = @_;
2744
2745     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2746
2747     return $1 || 1;
2748 }
2749
2750 sub config_to_command {
2751     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2752
2753     my $cmd = [];
2754     my $globalFlags = [];
2755     my $machineFlags = [];
2756     my $rtcFlags = [];
2757     my $cpuFlags = [];
2758     my $devices = [];
2759     my $pciaddr = '';
2760     my $bridges = {};
2761     my $kvmver = kvm_user_version();
2762     my $vernum = 0; # unknown
2763     my $ostype = $conf->{ostype};
2764     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2765         $vernum = $1*1000000+$2*1000;
2766     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2767         $vernum = $1*1000000+$2*1000+$3;
2768     }
2769
2770     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2771
2772     my $have_ovz = -f '/proc/vz/vestat';
2773
2774     my $q35 = machine_type_is_q35($conf);
2775     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2776     my $machine_type = $forcemachine || $conf->{machine};
2777     my $use_old_bios_files = undef;
2778     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2779
2780     my $cpuunits = defined($conf->{cpuunits}) ?
2781             $conf->{cpuunits} : $defaults->{cpuunits};
2782
2783     push @$cmd, '/usr/bin/kvm';
2784
2785     push @$cmd, '-id', $vmid;
2786
2787     my $use_virtio = 0;
2788
2789     my $qmpsocket = qmp_socket($vmid);
2790     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2791     push @$cmd, '-mon', "chardev=qmp,mode=control";
2792
2793
2794     push @$cmd, '-pidfile' , pidfile_name($vmid);
2795
2796     push @$cmd, '-daemonize';
2797
2798     if ($conf->{smbios1}) {
2799         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2800     }
2801
2802     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2803         my $ovmfbase;
2804
2805         # prefer the OVMF_CODE variant
2806         if (-f $OVMF_CODE) {
2807             $ovmfbase = $OVMF_CODE;
2808         } elsif (-f $OVMF_IMG) {
2809             $ovmfbase = $OVMF_IMG;
2810         }
2811
2812         die "no uefi base img found\n" if !$ovmfbase;
2813         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2814
2815         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2816             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2817             my $format = $d->{format} // 'raw';
2818             my $path;
2819             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2820             if ($storeid) {
2821                 $path = PVE::Storage::path($storecfg, $d->{file});
2822                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2823                 $format = qemu_img_format($scfg, $volname);
2824             } else {
2825                 $path = $d->{file};
2826                 $format = "raw";
2827             }
2828             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2829         } elsif ($ovmfbase eq $OVMF_CODE) {
2830             warn "using uefi without permanent efivars disk\n";
2831             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2832             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2833             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2834         } else {
2835             # if the base img is not OVMF_CODE, we do not have to bother
2836             # to create/use a vars image, since it will not be used anyway
2837             # this can only happen if someone manually deletes the OVMF_CODE image
2838             # or has an old pve-qemu-kvm version installed.
2839             # both should not happen, but we ignore it here
2840         }
2841     }
2842
2843
2844     # add usb controllers
2845     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2846     push @$devices, @usbcontrollers if @usbcontrollers;
2847     my $vga = $conf->{vga};
2848
2849     my $qxlnum = vga_conf_has_spice($vga);
2850     $vga = 'qxl' if $qxlnum;
2851
2852     if (!$vga) {
2853         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2854                                 $conf->{ostype} eq 'win7' ||
2855                                 $conf->{ostype} eq 'w2k8')) {
2856             $vga = 'std';
2857         } else {
2858             $vga = 'cirrus';
2859         }
2860     }
2861
2862     # enable absolute mouse coordinates (needed by vnc)
2863     my $tablet;
2864     if (defined($conf->{tablet})) {
2865         $tablet = $conf->{tablet};
2866     } else {
2867         $tablet = $defaults->{tablet};
2868         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2869         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2870     }
2871
2872     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2873
2874     my $kvm_off = 0;
2875     # host pci devices
2876     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2877         my $d = parse_hostpci($conf->{"hostpci$i"});
2878         next if !$d;
2879
2880         my $pcie = $d->{pcie};
2881         if($pcie){
2882             die "q35 machine model is not enabled" if !$q35;
2883             $pciaddr = print_pcie_addr("hostpci$i");
2884         }else{
2885             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2886         }
2887
2888         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2889         my $xvga = '';
2890         if ($d->{'x-vga'}) {
2891             $xvga = ',x-vga=on';
2892             $kvm_off = 1;
2893             $vga = 'none';
2894             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2895                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2896             }
2897             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2898                 $xvga = "";
2899             }
2900         }
2901         my $pcidevices = $d->{pciid};
2902         my $multifunction = 1 if @$pcidevices > 1;
2903
2904         my $j=0;
2905         foreach my $pcidevice (@$pcidevices) {
2906
2907             my $id = "hostpci$i";
2908             $id .= ".$j" if $multifunction;
2909             my $addr = $pciaddr;
2910             $addr .= ".$j" if $multifunction;
2911             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2912
2913             if($j == 0){
2914                 $devicestr .= "$rombar$xvga";
2915                 $devicestr .= ",multifunction=on" if $multifunction;
2916             }
2917
2918             push @$devices, '-device', $devicestr;
2919             $j++;
2920         }
2921     }
2922
2923     # usb devices
2924     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2925     push @$devices, @usbdevices if @usbdevices;
2926     # serial devices
2927     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2928         if (my $path = $conf->{"serial$i"}) {
2929             if ($path eq 'socket') {
2930                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2931                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2932                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2933             } else {
2934                 die "no such serial device\n" if ! -c $path;
2935                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2936                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2937             }
2938         }
2939     }
2940
2941     # parallel devices
2942     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2943         if (my $path = $conf->{"parallel$i"}) {
2944             die "no such parallel device\n" if ! -c $path;
2945             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2946             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2947             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2948         }
2949     }
2950
2951     my $vmname = $conf->{name} || "vm$vmid";
2952
2953     push @$cmd, '-name', $vmname;
2954
2955     my $sockets = 1;
2956     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2957     $sockets = $conf->{sockets} if  $conf->{sockets};
2958
2959     my $cores = $conf->{cores} || 1;
2960
2961     my $maxcpus = $sockets * $cores;
2962
2963     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2964
2965     my $allowed_vcpus = $cpuinfo->{cpus};
2966
2967     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2968         if ($allowed_vcpus < $maxcpus);
2969
2970     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
2971
2972         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2973         for (my $i = 2; $i <= $vcpus; $i++)  {
2974             my $cpustr = print_cpu_device($conf,$i);
2975             push @$cmd, '-device', $cpustr;
2976         }
2977
2978     } else {
2979
2980         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2981     }
2982     push @$cmd, '-nodefaults';
2983
2984     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2985
2986     my $bootindex_hash = {};
2987     my $i = 1;
2988     foreach my $o (split(//, $bootorder)) {
2989         $bootindex_hash->{$o} = $i*100;
2990         $i++;
2991     }
2992
2993     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
2994
2995     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2996
2997     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2998
2999     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3000
3001     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3002         my $socket = vnc_socket($vmid);
3003         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3004     } else {
3005         push @$cmd, '-nographic';
3006     }
3007
3008     # time drift fix
3009     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3010
3011     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3012     my $useLocaltime = $conf->{localtime};
3013
3014     if ($ostype) {
3015         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3016
3017         if ($ostype =~ m/^w/) { # windows
3018             $useLocaltime = 1 if !defined($conf->{localtime});
3019
3020             # use time drift fix when acpi is enabled
3021             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3022                 $tdf = 1 if !defined($conf->{tdf});
3023             }
3024         }
3025
3026         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3027             $ostype eq 'wvista') {
3028             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3029             push @$cmd, '-no-hpet';
3030             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3031                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3032                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3033                 push @$cpuFlags , 'hv_time' if !$nokvm;
3034
3035                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
3036                     push @$cpuFlags , 'hv_reset' if !$nokvm;
3037                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
3038                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
3039                 }
3040
3041             } else {
3042                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3043             }
3044         }
3045
3046         if ($ostype eq 'win7' || $ostype eq 'win8') {
3047             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3048         }
3049     }
3050
3051     push @$rtcFlags, 'driftfix=slew' if $tdf;
3052
3053     if ($nokvm) {
3054         push @$machineFlags, 'accel=tcg';
3055     } else {
3056         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3057     }
3058
3059     if ($machine_type) {
3060         push @$machineFlags, "type=${machine_type}";
3061     }
3062
3063     if ($conf->{startdate}) {
3064         push @$rtcFlags, "base=$conf->{startdate}";
3065     } elsif ($useLocaltime) {
3066         push @$rtcFlags, 'base=localtime';
3067     }
3068
3069     my $cpu = $nokvm ? "qemu64" : "kvm64";
3070     if (my $cputype = $conf->{cpu}) {
3071         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3072             or die "Cannot parse cpu description: $cputype\n";
3073         $cpu = $cpuconf->{cputype};
3074         $kvm_off = 1 if $cpuconf->{hidden};
3075     }
3076
3077     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3078
3079     push @$cpuFlags , '-x2apic'
3080         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3081
3082     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3083
3084     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3085
3086     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3087
3088         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3089         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3090     }
3091
3092     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3093
3094     push @$cpuFlags, 'kvm=off' if $kvm_off;
3095
3096     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3097         die "internal error"; # should not happen
3098
3099     push @$cpuFlags, "vendor=${cpu_vendor}"
3100         if $cpu_vendor ne 'default';
3101
3102     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3103
3104     push @$cmd, '-cpu', $cpu;
3105
3106     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3107
3108     push @$cmd, '-S' if $conf->{freeze};
3109
3110     # set keyboard layout
3111     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3112     push @$cmd, '-k', $kb if $kb;
3113
3114     # enable sound
3115     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3116     #push @$cmd, '-soundhw', 'es1370';
3117     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3118
3119     if($conf->{agent}) {
3120         my $qgasocket = qmp_socket($vmid, 1);
3121         my $pciaddr = print_pci_addr("qga0", $bridges);
3122         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3123         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3124         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3125     }
3126
3127     my $spice_port;
3128
3129     if ($qxlnum) {
3130         if ($qxlnum > 1) {
3131             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3132                 for(my $i = 1; $i < $qxlnum; $i++){
3133                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3134                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3135                 }
3136             } else {
3137                 # assume other OS works like Linux
3138                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3139                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3140             }
3141         }
3142
3143         my $pciaddr = print_pci_addr("spice", $bridges);
3144
3145         my $nodename = PVE::INotify::nodename();
3146         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3147         $spice_port = PVE::Tools::next_spice_port($pfamily);
3148
3149         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3150
3151         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3152         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3153         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3154     }
3155
3156     # enable balloon by default, unless explicitly disabled
3157     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3158         $pciaddr = print_pci_addr("balloon0", $bridges);
3159         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3160     }
3161
3162     if ($conf->{watchdog}) {
3163         my $wdopts = parse_watchdog($conf->{watchdog});
3164         $pciaddr = print_pci_addr("watchdog", $bridges);
3165         my $watchdog = $wdopts->{model} || 'i6300esb';
3166         push @$devices, '-device', "$watchdog$pciaddr";
3167         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3168     }
3169
3170     my $vollist = [];
3171     my $scsicontroller = {};
3172     my $ahcicontroller = {};
3173     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3174
3175     # Add iscsi initiator name if available
3176     if (my $initiator = get_initiator_name()) {
3177         push @$devices, '-iscsi', "initiator-name=$initiator";
3178     }
3179
3180     foreach_drive($conf, sub {
3181         my ($ds, $drive) = @_;
3182
3183         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3184             push @$vollist, $drive->{file};
3185         }
3186
3187         $use_virtio = 1 if $ds =~ m/^virtio/;
3188
3189         if (drive_is_cdrom ($drive)) {
3190             if ($bootindex_hash->{d}) {
3191                 $drive->{bootindex} = $bootindex_hash->{d};
3192                 $bootindex_hash->{d} += 1;
3193             }
3194         } else {
3195             if ($bootindex_hash->{c}) {
3196                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3197                 $bootindex_hash->{c} += 1;
3198             }
3199         }
3200
3201         if($drive->{interface} eq 'virtio'){
3202            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3203         }
3204
3205         if ($drive->{interface} eq 'scsi') {
3206
3207             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3208
3209             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3210             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3211
3212             my $iothread = '';
3213             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3214                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3215                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3216             } elsif ($drive->{iothread}) {
3217                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3218             }
3219
3220             my $queues = '';
3221             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3222                 $queues = ",num_queues=$drive->{queues}";
3223             }
3224
3225             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3226             $scsicontroller->{$controller}=1;
3227         }
3228
3229         if ($drive->{interface} eq 'sata') {
3230            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3231            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3232            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3233            $ahcicontroller->{$controller}=1;
3234         }
3235
3236         if ($drive->{interface} eq 'efidisk') {
3237             # this will be added somewhere else
3238             return;
3239         }
3240
3241         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3242         push @$devices, '-drive',$drive_cmd;
3243         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3244     });
3245
3246     for (my $i = 0; $i < $MAX_NETS; $i++) {
3247          next if !$conf->{"net$i"};
3248          my $d = parse_net($conf->{"net$i"});
3249          next if !$d;
3250
3251          $use_virtio = 1 if $d->{model} eq 'virtio';
3252
3253          if ($bootindex_hash->{n}) {
3254             $d->{bootindex} = $bootindex_hash->{n};
3255             $bootindex_hash->{n} += 1;
3256          }
3257
3258          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3259          push @$devices, '-netdev', $netdevfull;
3260
3261          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3262          push @$devices, '-device', $netdevicefull;
3263     }
3264
3265     if (!$q35) {
3266         # add pci bridges
3267         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3268            $bridges->{1} = 1;
3269            $bridges->{2} = 1;
3270         }
3271
3272         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3273
3274         while (my ($k, $v) = each %$bridges) {
3275             $pciaddr = print_pci_addr("pci.$k");
3276             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3277         }
3278     }
3279
3280     # add custom args
3281     if ($conf->{args}) {
3282         my $aa = PVE::Tools::split_args($conf->{args});
3283         push @$cmd, @$aa;
3284     }
3285
3286     push @$cmd, @$devices;
3287     push @$cmd, '-rtc', join(',', @$rtcFlags)
3288         if scalar(@$rtcFlags);
3289     push @$cmd, '-machine', join(',', @$machineFlags)
3290         if scalar(@$machineFlags);
3291     push @$cmd, '-global', join(',', @$globalFlags)
3292         if scalar(@$globalFlags);
3293
3294     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3295 }
3296
3297 sub vnc_socket {
3298     my ($vmid) = @_;
3299     return "${var_run_tmpdir}/$vmid.vnc";
3300 }
3301
3302 sub spice_port {
3303     my ($vmid) = @_;
3304
3305     my $res = vm_mon_cmd($vmid, 'query-spice');
3306
3307     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3308 }
3309
3310 sub qmp_socket {
3311     my ($vmid, $qga) = @_;
3312     my $sockettype = $qga ? 'qga' : 'qmp';
3313     return "${var_run_tmpdir}/$vmid.$sockettype";
3314 }
3315
3316 sub pidfile_name {
3317     my ($vmid) = @_;
3318     return "${var_run_tmpdir}/$vmid.pid";
3319 }
3320
3321 sub vm_devices_list {
3322     my ($vmid) = @_;
3323
3324     my $res = vm_mon_cmd($vmid, 'query-pci');
3325     my $devices = {};
3326     foreach my $pcibus (@$res) {
3327         foreach my $device (@{$pcibus->{devices}}) {
3328             next if !$device->{'qdev_id'};
3329             if ($device->{'pci_bridge'}) {
3330                 $devices->{$device->{'qdev_id'}} = 1;
3331                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3332                     next if !$bridge_device->{'qdev_id'};
3333                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3334                     $devices->{$device->{'qdev_id'}}++;
3335                 }
3336             } else {
3337                 $devices->{$device->{'qdev_id'}} = 1;
3338             }
3339         }
3340     }
3341
3342     my $resblock = vm_mon_cmd($vmid, 'query-block');
3343     foreach my $block (@$resblock) {
3344         if($block->{device} =~ m/^drive-(\S+)/){
3345                 $devices->{$1} = 1;
3346         }
3347     }
3348
3349     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3350     foreach my $mice (@$resmice) {
3351         if ($mice->{name} eq 'QEMU HID Tablet') {
3352             $devices->{tablet} = 1;
3353             last;
3354         }
3355     }
3356
3357     # for usb devices there is no query-usb
3358     # but we can iterate over the entries in
3359     # qom-list path=/machine/peripheral
3360     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3361     foreach my $per (@$resperipheral) {
3362         if ($per->{name} =~ m/^usb\d+$/) {
3363             $devices->{$per->{name}} = 1;
3364         }
3365     }
3366
3367     return $devices;
3368 }
3369
3370 sub vm_deviceplug {
3371     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3372
3373     my $q35 = machine_type_is_q35($conf);
3374
3375     my $devices_list = vm_devices_list($vmid);
3376     return 1 if defined($devices_list->{$deviceid});
3377
3378     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3379
3380     if ($deviceid eq 'tablet') {
3381
3382         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3383
3384     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3385
3386         die "usb hotplug currently not reliable\n";
3387         # since we can't reliably hot unplug all added usb devices
3388         # and usb passthrough disables live migration
3389         # we disable usb hotplugging for now
3390         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3391
3392     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3393
3394         qemu_iothread_add($vmid, $deviceid, $device);
3395
3396         qemu_driveadd($storecfg, $vmid, $device);
3397         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3398
3399         qemu_deviceadd($vmid, $devicefull);
3400         eval { qemu_deviceaddverify($vmid, $deviceid); };
3401         if (my $err = $@) {
3402             eval { qemu_drivedel($vmid, $deviceid); };
3403             warn $@ if $@;
3404             die $err;
3405         }
3406
3407     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3408
3409
3410         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3411         my $pciaddr = print_pci_addr($deviceid);
3412         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3413
3414         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3415
3416         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3417             qemu_iothread_add($vmid, $deviceid, $device);
3418             $devicefull .= ",iothread=iothread-$deviceid";
3419         }
3420
3421         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3422             $devicefull .= ",num_queues=$device->{queues}";
3423         }
3424
3425         qemu_deviceadd($vmid, $devicefull);
3426         qemu_deviceaddverify($vmid, $deviceid);
3427
3428     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3429
3430         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3431         qemu_driveadd($storecfg, $vmid, $device);
3432
3433         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3434         eval { qemu_deviceadd($vmid, $devicefull); };
3435         if (my $err = $@) {
3436             eval { qemu_drivedel($vmid, $deviceid); };
3437             warn $@ if $@;
3438             die $err;
3439         }
3440
3441     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3442
3443         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3444
3445         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3446         my $use_old_bios_files = undef;
3447         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3448
3449         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3450         qemu_deviceadd($vmid, $netdevicefull);
3451         eval { qemu_deviceaddverify($vmid, $deviceid); };
3452         if (my $err = $@) {
3453             eval { qemu_netdevdel($vmid, $deviceid); };
3454             warn $@ if $@;
3455             die $err;
3456         }
3457
3458     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3459
3460         my $bridgeid = $2;
3461         my $pciaddr = print_pci_addr($deviceid);
3462         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3463
3464         qemu_deviceadd($vmid, $devicefull);
3465         qemu_deviceaddverify($vmid, $deviceid);
3466
3467     } else {
3468         die "can't hotplug device '$deviceid'\n";
3469     }
3470
3471     return 1;
3472 }
3473
3474 # fixme: this should raise exceptions on error!
3475 sub vm_deviceunplug {
3476     my ($vmid, $conf, $deviceid) = @_;
3477
3478     my $devices_list = vm_devices_list($vmid);
3479     return 1 if !defined($devices_list->{$deviceid});
3480
3481     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3482
3483     if ($deviceid eq 'tablet') {
3484
3485         qemu_devicedel($vmid, $deviceid);
3486
3487     } elsif ($deviceid =~ m/^usb\d+$/) {
3488
3489         die "usb hotplug currently not reliable\n";
3490         # when unplugging usb devices this way,
3491         # there may be remaining usb controllers/hubs
3492         # so we disable it for now
3493         qemu_devicedel($vmid, $deviceid);
3494         qemu_devicedelverify($vmid, $deviceid);
3495
3496     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3497
3498         qemu_devicedel($vmid, $deviceid);
3499         qemu_devicedelverify($vmid, $deviceid);
3500         qemu_drivedel($vmid, $deviceid);
3501         qemu_iothread_del($conf, $vmid, $deviceid);
3502
3503     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3504
3505         qemu_devicedel($vmid, $deviceid);
3506         qemu_devicedelverify($vmid, $deviceid);
3507         qemu_iothread_del($conf, $vmid, $deviceid);
3508
3509     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3510
3511         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3512         my $device = parse_drive($deviceid, $conf->{$deviceid});
3513         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3514
3515         qemu_devicedel($vmid, $deviceid);
3516         qemu_drivedel($vmid, $deviceid);
3517         qemu_deletescsihw($conf, $vmid, $deviceid);
3518
3519     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3520
3521         qemu_devicedel($vmid, $deviceid);
3522         qemu_devicedelverify($vmid, $deviceid);
3523         qemu_netdevdel($vmid, $deviceid);
3524
3525     } else {
3526         die "can't unplug device '$deviceid'\n";
3527     }
3528
3529     return 1;
3530 }
3531
3532 sub qemu_deviceadd {
3533     my ($vmid, $devicefull) = @_;
3534
3535     $devicefull = "driver=".$devicefull;
3536     my %options =  split(/[=,]/, $devicefull);
3537
3538     vm_mon_cmd($vmid, "device_add" , %options);
3539 }
3540
3541 sub qemu_devicedel {
3542     my ($vmid, $deviceid) = @_;
3543
3544     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3545 }
3546
3547 sub qemu_iothread_add {
3548     my($vmid, $deviceid, $device) = @_;
3549
3550     if ($device->{iothread}) {
3551         my $iothreads = vm_iothreads_list($vmid);
3552         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3553     }
3554 }
3555
3556 sub qemu_iothread_del {
3557     my($conf, $vmid, $deviceid) = @_;
3558
3559     my $device = parse_drive($deviceid, $conf->{$deviceid});
3560     if ($device->{iothread}) {
3561         my $iothreads = vm_iothreads_list($vmid);
3562         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3563     }
3564 }
3565
3566 sub qemu_objectadd {
3567     my($vmid, $objectid, $qomtype) = @_;
3568
3569     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3570
3571     return 1;
3572 }
3573
3574 sub qemu_objectdel {
3575     my($vmid, $objectid) = @_;
3576
3577     vm_mon_cmd($vmid, "object-del", id => $objectid);
3578
3579     return 1;
3580 }
3581
3582 sub qemu_driveadd {
3583     my ($storecfg, $vmid, $device) = @_;
3584
3585     my $drive = print_drive_full($storecfg, $vmid, $device);
3586     $drive =~ s/\\/\\\\/g;
3587     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3588
3589     # If the command succeeds qemu prints: "OK"
3590     return 1 if $ret =~ m/OK/s;
3591
3592     die "adding drive failed: $ret\n";
3593 }
3594
3595 sub qemu_drivedel {
3596     my($vmid, $deviceid) = @_;
3597
3598     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3599     $ret =~ s/^\s+//;
3600
3601     return 1 if $ret eq "";
3602
3603     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3604     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3605
3606     die "deleting drive $deviceid failed : $ret\n";
3607 }
3608
3609 sub qemu_deviceaddverify {
3610     my ($vmid, $deviceid) = @_;
3611
3612     for (my $i = 0; $i <= 5; $i++) {
3613          my $devices_list = vm_devices_list($vmid);
3614          return 1 if defined($devices_list->{$deviceid});
3615          sleep 1;
3616     }
3617
3618     die "error on hotplug device '$deviceid'\n";
3619 }
3620
3621
3622 sub qemu_devicedelverify {
3623     my ($vmid, $deviceid) = @_;
3624
3625     # need to verify that the device is correctly removed as device_del
3626     # is async and empty return is not reliable
3627
3628     for (my $i = 0; $i <= 5; $i++) {
3629          my $devices_list = vm_devices_list($vmid);
3630          return 1 if !defined($devices_list->{$deviceid});
3631          sleep 1;
3632     }
3633
3634     die "error on hot-unplugging device '$deviceid'\n";
3635 }
3636
3637 sub qemu_findorcreatescsihw {
3638     my ($storecfg, $conf, $vmid, $device) = @_;
3639
3640     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3641
3642     my $scsihwid="$controller_prefix$controller";
3643     my $devices_list = vm_devices_list($vmid);
3644
3645     if(!defined($devices_list->{$scsihwid})) {
3646         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3647     }
3648
3649     return 1;
3650 }
3651
3652 sub qemu_deletescsihw {
3653     my ($conf, $vmid, $opt) = @_;
3654
3655     my $device = parse_drive($opt, $conf->{$opt});
3656
3657     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3658         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3659         return 1;
3660     }
3661
3662     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3663
3664     my $devices_list = vm_devices_list($vmid);
3665     foreach my $opt (keys %{$devices_list}) {
3666         if (PVE::QemuServer::is_valid_drivename($opt)) {
3667             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3668             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3669                 return 1;
3670             }
3671         }
3672     }
3673
3674     my $scsihwid="scsihw$controller";
3675
3676     vm_deviceunplug($vmid, $conf, $scsihwid);
3677
3678     return 1;
3679 }
3680
3681 sub qemu_add_pci_bridge {
3682     my ($storecfg, $conf, $vmid, $device) = @_;
3683
3684     my $bridges = {};
3685
3686     my $bridgeid;
3687
3688     print_pci_addr($device, $bridges);
3689
3690     while (my ($k, $v) = each %$bridges) {
3691         $bridgeid = $k;
3692     }
3693     return 1 if !defined($bridgeid) || $bridgeid < 1;
3694
3695     my $bridge = "pci.$bridgeid";
3696     my $devices_list = vm_devices_list($vmid);
3697
3698     if (!defined($devices_list->{$bridge})) {
3699         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3700     }
3701
3702     return 1;
3703 }
3704
3705 sub qemu_set_link_status {
3706     my ($vmid, $device, $up) = @_;
3707
3708     vm_mon_cmd($vmid, "set_link", name => $device,
3709                up => $up ? JSON::true : JSON::false);
3710 }
3711
3712 sub qemu_netdevadd {
3713     my ($vmid, $conf, $device, $deviceid) = @_;
3714
3715     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3716     my %options =  split(/[=,]/, $netdev);
3717
3718     vm_mon_cmd($vmid, "netdev_add",  %options);
3719     return 1;
3720 }
3721
3722 sub qemu_netdevdel {
3723     my ($vmid, $deviceid) = @_;
3724
3725     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3726 }
3727
3728 sub qemu_usb_hotplug {
3729     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3730
3731     return if !$device;
3732
3733     # remove the old one first
3734     vm_deviceunplug($vmid, $conf, $deviceid);
3735
3736     # check if xhci controller is necessary and available
3737     if ($device->{usb3}) {
3738
3739         my $devicelist = vm_devices_list($vmid);
3740
3741         if (!$devicelist->{xhci}) {
3742             my $pciaddr = print_pci_addr("xhci");
3743             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3744         }
3745     }
3746     my $d = parse_usb_device($device->{host});
3747     $d->{usb3} = $device->{usb3};
3748
3749     # add the new one
3750     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3751 }
3752
3753 sub qemu_cpu_hotplug {
3754     my ($vmid, $conf, $vcpus) = @_;
3755
3756     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3757
3758     my $sockets = 1;
3759     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3760     $sockets = $conf->{sockets} if  $conf->{sockets};
3761     my $cores = $conf->{cores} || 1;
3762     my $maxcpus = $sockets * $cores;
3763
3764     $vcpus = $maxcpus if !$vcpus;
3765
3766     die "you can't add more vcpus than maxcpus\n"
3767         if $vcpus > $maxcpus;
3768
3769     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3770
3771     if ($vcpus < $currentvcpus) {
3772
3773         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3774
3775             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3776                 qemu_devicedel($vmid, "cpu$i");
3777                 my $retry = 0;
3778                 my $currentrunningvcpus = undef;
3779                 while (1) {
3780                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3781                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3782                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3783                     $retry++;
3784                     sleep 1;
3785                 }
3786                 #update conf after each succesfull cpu unplug
3787                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3788                 PVE::QemuConfig->write_config($vmid, $conf);
3789             }
3790         } else {
3791             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3792         }
3793
3794         return;
3795     }
3796
3797     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3798     die "vcpus in running vm does not match its configuration\n"
3799         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3800
3801     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3802
3803         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3804             my $cpustr = print_cpu_device($conf, $i);
3805             qemu_deviceadd($vmid, $cpustr);
3806
3807             my $retry = 0;
3808             my $currentrunningvcpus = undef;
3809             while (1) {
3810                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3811                 last if scalar(@{$currentrunningvcpus}) == $i;
3812                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3813                 sleep 1;
3814                 $retry++;
3815             }
3816             #update conf after each succesfull cpu hotplug
3817             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3818             PVE::QemuConfig->write_config($vmid, $conf);
3819         }
3820     } else {
3821
3822         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3823             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3824         }
3825     }
3826 }
3827
3828 sub qemu_block_set_io_throttle {
3829     my ($vmid, $deviceid,
3830         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3831         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3832
3833     return if !check_running($vmid) ;
3834
3835     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3836         bps => int($bps),
3837         bps_rd => int($bps_rd),
3838         bps_wr => int($bps_wr),
3839         iops => int($iops),
3840         iops_rd => int($iops_rd),
3841         iops_wr => int($iops_wr),
3842         bps_max => int($bps_max),
3843         bps_rd_max => int($bps_rd_max),
3844         bps_wr_max => int($bps_wr_max),
3845         iops_max => int($iops_max),
3846         iops_rd_max => int($iops_rd_max),
3847         iops_wr_max => int($iops_wr_max)
3848     );
3849
3850 }
3851
3852 # old code, only used to shutdown old VM after update
3853 sub __read_avail {
3854     my ($fh, $timeout) = @_;
3855
3856     my $sel = new IO::Select;
3857     $sel->add($fh);
3858
3859     my $res = '';
3860     my $buf;
3861
3862     my @ready;
3863     while (scalar (@ready = $sel->can_read($timeout))) {
3864         my $count;
3865         if ($count = $fh->sysread($buf, 8192)) {
3866             if ($buf =~ /^(.*)\(qemu\) $/s) {
3867                 $res .= $1;
3868                 last;
3869             } else {
3870                 $res .= $buf;
3871             }
3872         } else {
3873             if (!defined($count)) {
3874                 die "$!\n";
3875             }
3876             last;
3877         }
3878     }
3879
3880     die "monitor read timeout\n" if !scalar(@ready);
3881
3882     return $res;
3883 }
3884
3885 # old code, only used to shutdown old VM after update
3886 sub vm_monitor_command {
3887     my ($vmid, $cmdstr, $nocheck) = @_;
3888
3889     my $res;
3890
3891     eval {
3892         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3893
3894         my $sname = "${var_run_tmpdir}/$vmid.mon";
3895
3896         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3897             die "unable to connect to VM $vmid socket - $!\n";
3898
3899         my $timeout = 3;
3900
3901         # hack: migrate sometime blocks the monitor (when migrate_downtime
3902         # is set)
3903         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3904             $timeout = 60*60; # 1 hour
3905         }
3906
3907         # read banner;
3908         my $data = __read_avail($sock, $timeout);
3909
3910         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3911             die "got unexpected qemu monitor banner\n";
3912         }
3913
3914         my $sel = new IO::Select;
3915         $sel->add($sock);
3916
3917         if (!scalar(my @ready = $sel->can_write($timeout))) {
3918             die "monitor write error - timeout";
3919         }
3920
3921         my $fullcmd = "$cmdstr\r";
3922
3923         # syslog('info', "VM $vmid monitor command: $cmdstr");
3924
3925         my $b;
3926         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3927             die "monitor write error - $!";
3928         }
3929
3930         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3931
3932         $timeout = 20;
3933
3934         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3935             $timeout = 60*60; # 1 hour
3936         } elsif ($cmdstr =~ m/^(eject|change)/) {
3937             $timeout = 60; # note: cdrom mount command is slow
3938         }
3939         if ($res = __read_avail($sock, $timeout)) {
3940
3941             my @lines = split("\r?\n", $res);
3942
3943             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3944
3945             $res = join("\n", @lines);
3946             $res .= "\n";
3947         }
3948     };
3949
3950     my $err = $@;
3951
3952     if ($err) {
3953         syslog("err", "VM $vmid monitor command failed - $err");
3954         die $err;
3955     }
3956
3957     return $res;
3958 }
3959
3960 sub qemu_block_resize {
3961     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3962
3963     my $running = check_running($vmid);
3964
3965     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3966
3967     return if !$running;
3968
3969     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3970
3971 }
3972
3973 sub qemu_volume_snapshot {
3974     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3975
3976     my $running = check_running($vmid);
3977
3978     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3979         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3980     } else {
3981         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3982     }
3983 }
3984
3985 sub qemu_volume_snapshot_delete {
3986     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3987
3988     my $running = check_running($vmid);
3989
3990     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3991
3992     return if !$running;
3993
3994     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3995 }
3996
3997 sub set_migration_caps {
3998     my ($vmid) = @_;
3999
4000     my $cap_ref = [];
4001
4002     my $enabled_cap = {
4003         "auto-converge" => 1,
4004         "xbzrle" => 1,
4005         "x-rdma-pin-all" => 0,
4006         "zero-blocks" => 0,
4007         "compress" => 0
4008     };
4009
4010     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4011
4012     for my $supported_capability (@$supported_capabilities) {
4013         push @$cap_ref, {
4014             capability => $supported_capability->{capability},
4015             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4016         };
4017     }
4018
4019     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4020 }
4021
4022 my $fast_plug_option = {
4023     'lock' => 1,
4024     'name' => 1,
4025     'onboot' => 1,
4026     'shares' => 1,
4027     'startup' => 1,
4028     'description' => 1,
4029     'protection' => 1,
4030 };
4031
4032 # hotplug changes in [PENDING]
4033 # $selection hash can be used to only apply specified options, for
4034 # example: { cores => 1 } (only apply changed 'cores')
4035 # $errors ref is used to return error messages
4036 sub vmconfig_hotplug_pending {
4037     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4038
4039     my $defaults = load_defaults();
4040
4041     # commit values which do not have any impact on running VM first
4042     # Note: those option cannot raise errors, we we do not care about
4043     # $selection and always apply them.
4044
4045     my $add_error = sub {
4046         my ($opt, $msg) = @_;
4047         $errors->{$opt} = "hotplug problem - $msg";
4048     };
4049
4050     my $changes = 0;
4051     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4052         if ($fast_plug_option->{$opt}) {
4053             $conf->{$opt} = $conf->{pending}->{$opt};
4054             delete $conf->{pending}->{$opt};
4055             $changes = 1;
4056         }
4057     }
4058
4059     if ($changes) {
4060         PVE::QemuConfig->write_config($vmid, $conf);
4061         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4062     }
4063
4064     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4065
4066     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4067     while (my ($opt, $force) = each %$pending_delete_hash) {
4068         next if $selection && !$selection->{$opt};
4069         eval {
4070             if ($opt eq 'hotplug') {
4071                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4072             } elsif ($opt eq 'tablet') {
4073                 die "skip\n" if !$hotplug_features->{usb};
4074                 if ($defaults->{tablet}) {
4075                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4076                 } else {
4077                     vm_deviceunplug($vmid, $conf, $opt);
4078                 }
4079             } elsif ($opt =~ m/^usb\d+/) {
4080                 die "skip\n";
4081                 # since we cannot reliably hot unplug usb devices
4082                 # we are disabling it
4083                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4084                 vm_deviceunplug($vmid, $conf, $opt);
4085             } elsif ($opt eq 'vcpus') {
4086                 die "skip\n" if !$hotplug_features->{cpu};
4087                 qemu_cpu_hotplug($vmid, $conf, undef);
4088             } elsif ($opt eq 'balloon') {
4089                 # enable balloon device is not hotpluggable
4090                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4091             } elsif ($fast_plug_option->{$opt}) {
4092                 # do nothing
4093             } elsif ($opt =~ m/^net(\d+)$/) {
4094                 die "skip\n" if !$hotplug_features->{network};
4095                 vm_deviceunplug($vmid, $conf, $opt);
4096             } elsif (is_valid_drivename($opt)) {
4097                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4098                 vm_deviceunplug($vmid, $conf, $opt);
4099                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4100             } elsif ($opt =~ m/^memory$/) {
4101                 die "skip\n" if !$hotplug_features->{memory};
4102                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4103             } elsif ($opt eq 'cpuunits') {
4104                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4105             } elsif ($opt eq 'cpulimit') {
4106                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4107             } else {
4108                 die "skip\n";
4109             }
4110         };
4111         if (my $err = $@) {
4112             &$add_error($opt, $err) if $err ne "skip\n";
4113         } else {
4114             # save new config if hotplug was successful
4115             delete $conf->{$opt};
4116             vmconfig_undelete_pending_option($conf, $opt);
4117             PVE::QemuConfig->write_config($vmid, $conf);
4118             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4119         }
4120     }
4121
4122     foreach my $opt (keys %{$conf->{pending}}) {
4123         next if $selection && !$selection->{$opt};
4124         my $value = $conf->{pending}->{$opt};
4125         eval {
4126             if ($opt eq 'hotplug') {
4127                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4128             } elsif ($opt eq 'tablet') {
4129                 die "skip\n" if !$hotplug_features->{usb};
4130                 if ($value == 1) {
4131                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4132                 } elsif ($value == 0) {
4133                     vm_deviceunplug($vmid, $conf, $opt);
4134                 }
4135             } elsif ($opt =~ m/^usb\d+$/) {
4136                 die "skip\n";
4137                 # since we cannot reliably hot unplug usb devices
4138                 # we are disabling it
4139                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4140                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4141                 die "skip\n" if !$d;
4142                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4143             } elsif ($opt eq 'vcpus') {
4144                 die "skip\n" if !$hotplug_features->{cpu};
4145                 qemu_cpu_hotplug($vmid, $conf, $value);
4146             } elsif ($opt eq 'balloon') {
4147                 # enable/disable balloning device is not hotpluggable
4148                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4149                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4150                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4151
4152                 # allow manual ballooning if shares is set to zero
4153                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4154                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4155                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4156                 }
4157             } elsif ($opt =~ m/^net(\d+)$/) {
4158                 # some changes can be done without hotplug
4159                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4160                                     $vmid, $opt, $value);
4161             } elsif (is_valid_drivename($opt)) {
4162                 # some changes can be done without hotplug
4163                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4164                                      $vmid, $opt, $value, 1);
4165             } elsif ($opt =~ m/^memory$/) { #dimms
4166                 die "skip\n" if !$hotplug_features->{memory};
4167                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4168             } elsif ($opt eq 'cpuunits') {
4169                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4170             } elsif ($opt eq 'cpulimit') {
4171                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4172                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4173             } else {
4174                 die "skip\n";  # skip non-hot-pluggable options
4175             }
4176         };
4177         if (my $err = $@) {
4178             &$add_error($opt, $err) if $err ne "skip\n";
4179         } else {
4180             # save new config if hotplug was successful
4181             $conf->{$opt} = $value;
4182             delete $conf->{pending}->{$opt};
4183             PVE::QemuConfig->write_config($vmid, $conf);
4184             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4185         }
4186     }
4187 }
4188
4189 sub try_deallocate_drive {
4190     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4191
4192     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4193         my $volid = $drive->{file};
4194         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4195             my $sid = PVE::Storage::parse_volume_id($volid);
4196             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4197
4198             # check if the disk is really unused
4199             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4200                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4201             PVE::Storage::vdisk_free($storecfg, $volid);
4202             return 1;
4203         } else {
4204             # If vm is not owner of this disk remove from config
4205             return 1;
4206         }
4207     }
4208
4209     return undef;
4210 }
4211
4212 sub vmconfig_delete_or_detach_drive {
4213     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4214
4215     my $drive = parse_drive($opt, $conf->{$opt});
4216
4217     my $rpcenv = PVE::RPCEnvironment::get();
4218     my $authuser = $rpcenv->get_user();
4219
4220     if ($force) {
4221         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4222         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4223     } else {
4224         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4225     }
4226 }
4227
4228 sub vmconfig_apply_pending {
4229     my ($vmid, $conf, $storecfg) = @_;
4230
4231     # cold plug
4232
4233     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4234     while (my ($opt, $force) = each %$pending_delete_hash) {
4235         die "internal error" if $opt =~ m/^unused/;
4236         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4237         if (!defined($conf->{$opt})) {
4238             vmconfig_undelete_pending_option($conf, $opt);
4239             PVE::QemuConfig->write_config($vmid, $conf);
4240         } elsif (is_valid_drivename($opt)) {
4241             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4242             vmconfig_undelete_pending_option($conf, $opt);
4243             delete $conf->{$opt};
4244             PVE::QemuConfig->write_config($vmid, $conf);
4245         } else {
4246             vmconfig_undelete_pending_option($conf, $opt);
4247             delete $conf->{$opt};
4248             PVE::QemuConfig->write_config($vmid, $conf);
4249         }
4250     }
4251
4252     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4253
4254     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4255         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4256
4257         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4258             # skip if nothing changed
4259         } elsif (is_valid_drivename($opt)) {
4260             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4261                 if defined($conf->{$opt});
4262             $conf->{$opt} = $conf->{pending}->{$opt};
4263         } else {
4264             $conf->{$opt} = $conf->{pending}->{$opt};
4265         }
4266
4267         delete $conf->{pending}->{$opt};
4268         PVE::QemuConfig->write_config($vmid, $conf);
4269     }
4270 }
4271
4272 my $safe_num_ne = sub {
4273     my ($a, $b) = @_;
4274
4275     return 0 if !defined($a) && !defined($b);
4276     return 1 if !defined($a);
4277     return 1 if !defined($b);
4278
4279     return $a != $b;
4280 };
4281
4282 my $safe_string_ne = sub {
4283     my ($a, $b) = @_;
4284
4285     return 0 if !defined($a) && !defined($b);
4286     return 1 if !defined($a);
4287     return 1 if !defined($b);
4288
4289     return $a ne $b;
4290 };
4291
4292 sub vmconfig_update_net {
4293     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4294
4295     my $newnet = parse_net($value);
4296
4297     if ($conf->{$opt}) {
4298         my $oldnet = parse_net($conf->{$opt});
4299
4300         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4301             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4302             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4303             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4304
4305             # for non online change, we try to hot-unplug
4306             die "skip\n" if !$hotplug;
4307             vm_deviceunplug($vmid, $conf, $opt);
4308         } else {
4309
4310             die "internal error" if $opt !~ m/net(\d+)/;
4311             my $iface = "tap${vmid}i$1";
4312
4313             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4314                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4315                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4316                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4317                 PVE::Network::tap_unplug($iface);
4318                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4319             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4320                 # Rate can be applied on its own but any change above needs to
4321                 # include the rate in tap_plug since OVS resets everything.
4322                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4323             }
4324
4325             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4326                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4327             }
4328
4329             return 1;
4330         }
4331     }
4332
4333     if ($hotplug) {
4334         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4335     } else {
4336         die "skip\n";
4337     }
4338 }
4339
4340 sub vmconfig_update_disk {
4341     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4342
4343     # fixme: do we need force?
4344
4345     my $drive = parse_drive($opt, $value);
4346
4347     if ($conf->{$opt}) {
4348
4349         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4350
4351             my $media = $drive->{media} || 'disk';
4352             my $oldmedia = $old_drive->{media} || 'disk';
4353             die "unable to change media type\n" if $media ne $oldmedia;
4354
4355             if (!drive_is_cdrom($old_drive)) {
4356
4357                 if ($drive->{file} ne $old_drive->{file}) {
4358
4359                     die "skip\n" if !$hotplug;
4360
4361                     # unplug and register as unused
4362                     vm_deviceunplug($vmid, $conf, $opt);
4363                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4364
4365                 } else {
4366                     # update existing disk
4367
4368                     # skip non hotpluggable value
4369                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4370                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4371                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4372                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4373                         die "skip\n";
4374                     }
4375
4376                     # apply throttle
4377                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4378                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4379                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4380                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4381                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4382                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4383                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4384                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4385                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4386                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4387                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4388                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4389
4390                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4391                                                    ($drive->{mbps} || 0)*1024*1024,
4392                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4393                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4394                                                    $drive->{iops} || 0,
4395                                                    $drive->{iops_rd} || 0,
4396                                                    $drive->{iops_wr} || 0,
4397                                                    ($drive->{mbps_max} || 0)*1024*1024,
4398                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4399                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4400                                                    $drive->{iops_max} || 0,
4401                                                    $drive->{iops_rd_max} || 0,
4402                                                    $drive->{iops_wr_max} || 0);
4403
4404                     }
4405
4406                     return 1;
4407                 }
4408
4409             } else { # cdrom
4410
4411                 if ($drive->{file} eq 'none') {
4412                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4413                 } else {
4414                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4415                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4416                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4417                 }
4418
4419                 return 1;
4420             }
4421         }
4422     }
4423
4424     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4425     # hotplug new disks
4426     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4427     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4428 }
4429
4430 sub vm_start {
4431     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4432         $forcemachine, $spice_ticket, $migration_network, $migration_type) = @_;
4433
4434     PVE::QemuConfig->lock_config($vmid, sub {
4435         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4436
4437         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4438
4439         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4440
4441         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4442
4443         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4444             vmconfig_apply_pending($vmid, $conf, $storecfg);
4445             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4446         }
4447
4448         my $defaults = load_defaults();
4449
4450         # set environment variable useful inside network script
4451         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4452
4453         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4454
4455         my $migrate_port = 0;
4456         my $migrate_uri;
4457         if ($statefile) {
4458             if ($statefile eq 'tcp') {
4459                 my $localip = "localhost";
4460                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4461                 my $nodename = PVE::INotify::nodename();
4462
4463                 if ($migration_type eq 'insecure') {
4464                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4465                     if ($migrate_network_addr) {
4466                         $localip = $migrate_network_addr;
4467                     } else {
4468                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4469                     }
4470
4471                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4472                 }
4473
4474                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4475                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4476                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4477                 push @$cmd, '-incoming', $migrate_uri;
4478                 push @$cmd, '-S';
4479
4480             } elsif ($statefile eq 'unix') {
4481                 # should be default for secure migrations as a ssh TCP forward
4482                 # tunnel is not deterministic reliable ready and fails regurarly
4483                 # to set up in time, so use UNIX socket forwards
4484                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4485                 unlink $socket_addr;
4486
4487                 $migrate_uri = "unix:$socket_addr";
4488
4489                 push @$cmd, '-incoming', $migrate_uri;
4490                 push @$cmd, '-S';
4491
4492             } else {
4493                 push @$cmd, '-loadstate', $statefile;
4494             }
4495         } elsif ($paused) {
4496             push @$cmd, '-S';
4497         }
4498
4499         # host pci devices
4500         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4501           my $d = parse_hostpci($conf->{"hostpci$i"});
4502           next if !$d;
4503           my $pcidevices = $d->{pciid};
4504           foreach my $pcidevice (@$pcidevices) {
4505                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4506
4507                 my $info = pci_device_info("0000:$pciid");
4508                 die "IOMMU not present\n" if !check_iommu_support();
4509                 die "no pci device info for device '$pciid'\n" if !$info;
4510                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4511                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4512           }
4513         }
4514
4515         PVE::Storage::activate_volumes($storecfg, $vollist);
4516
4517         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4518             my $cmd = [];
4519             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4520             eval  { run_command($cmd); };
4521         }
4522
4523         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4524                                                   : $defaults->{cpuunits};
4525
4526         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4527
4528         my %properties = (
4529             Slice => 'qemu.slice',
4530             KillMode => 'none',
4531             CPUShares => $cpuunits
4532         );
4533
4534         if (my $cpulimit = $conf->{cpulimit}) {
4535             $properties{CPUQuota} = int($cpulimit * 100);
4536         }
4537         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4538
4539         if ($conf->{hugepages}) {
4540
4541             my $code = sub {
4542                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4543                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4544
4545                 PVE::QemuServer::Memory::hugepages_mount();
4546                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4547
4548                 eval  {
4549                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4550                     run_command($cmd, %run_params);
4551                 };
4552
4553                 if (my $err = $@) {
4554                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4555                     die $err;
4556                 }
4557
4558                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4559             };
4560             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4561
4562         } else {
4563             eval  {
4564                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4565                 run_command($cmd, %run_params);
4566             };
4567         }
4568
4569         if (my $err = $@) {
4570             # deactivate volumes if start fails
4571             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4572             die "start failed: $err";
4573         }
4574
4575         print "migration listens on $migrate_uri\n" if $migrate_uri;
4576
4577         if ($statefile && $statefile ne 'tcp')  {
4578             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4579             warn $@ if $@;
4580         }
4581
4582         if ($migratedfrom) {
4583
4584             eval {
4585                 set_migration_caps($vmid);
4586             };
4587             warn $@ if $@;
4588
4589             if ($spice_port) {
4590                 print "spice listens on port $spice_port\n";
4591                 if ($spice_ticket) {
4592                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4593                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4594                 }
4595             }
4596
4597         } else {
4598
4599             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4600                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4601                     if $conf->{balloon};
4602             }
4603
4604             foreach my $opt (keys %$conf) {
4605                 next if $opt !~  m/^net\d+$/;
4606                 my $nicconf = parse_net($conf->{$opt});
4607                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4608             }
4609         }
4610
4611         vm_mon_cmd_nocheck($vmid, 'qom-set',
4612                     path => "machine/peripheral/balloon0",
4613                     property => "guest-stats-polling-interval",
4614                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4615
4616     });
4617 }
4618
4619 sub vm_mon_cmd {
4620     my ($vmid, $execute, %params) = @_;
4621
4622     my $cmd = { execute => $execute, arguments => \%params };
4623     vm_qmp_command($vmid, $cmd);
4624 }
4625
4626 sub vm_mon_cmd_nocheck {
4627     my ($vmid, $execute, %params) = @_;
4628
4629     my $cmd = { execute => $execute, arguments => \%params };
4630     vm_qmp_command($vmid, $cmd, 1);
4631 }
4632
4633 sub vm_qmp_command {
4634     my ($vmid, $cmd, $nocheck) = @_;
4635
4636     my $res;
4637
4638     my $timeout;
4639     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4640         $timeout = $cmd->{arguments}->{timeout};
4641         delete $cmd->{arguments}->{timeout};
4642     }
4643
4644     eval {
4645         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4646         my $sname = qmp_socket($vmid);
4647         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4648             my $qmpclient = PVE::QMPClient->new();
4649
4650             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4651         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4652             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4653                 if scalar(%{$cmd->{arguments}});
4654             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4655         } else {
4656             die "unable to open monitor socket\n";
4657         }
4658     };
4659     if (my $err = $@) {
4660         syslog("err", "VM $vmid qmp command failed - $err");
4661         die $err;
4662     }
4663
4664     return $res;
4665 }
4666
4667 sub vm_human_monitor_command {
4668     my ($vmid, $cmdline) = @_;
4669
4670     my $res;
4671
4672     my $cmd = {
4673         execute => 'human-monitor-command',
4674         arguments => { 'command-line' => $cmdline},
4675     };
4676
4677     return vm_qmp_command($vmid, $cmd);
4678 }
4679
4680 sub vm_commandline {
4681     my ($storecfg, $vmid) = @_;
4682
4683     my $conf = PVE::QemuConfig->load_config($vmid);
4684
4685     my $defaults = load_defaults();
4686
4687     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4688
4689     return PVE::Tools::cmd2string($cmd);
4690 }
4691
4692 sub vm_reset {
4693     my ($vmid, $skiplock) = @_;
4694
4695     PVE::QemuConfig->lock_config($vmid, sub {
4696
4697         my $conf = PVE::QemuConfig->load_config($vmid);
4698
4699         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4700
4701         vm_mon_cmd($vmid, "system_reset");
4702     });
4703 }
4704
4705 sub get_vm_volumes {
4706     my ($conf) = @_;
4707
4708     my $vollist = [];
4709     foreach_volid($conf, sub {
4710         my ($volid, $is_cdrom) = @_;
4711
4712         return if $volid =~ m|^/|;
4713
4714         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4715         return if !$sid;
4716
4717         push @$vollist, $volid;
4718     });
4719
4720     return $vollist;
4721 }
4722
4723 sub vm_stop_cleanup {
4724     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4725
4726     eval {
4727
4728         if (!$keepActive) {
4729             my $vollist = get_vm_volumes($conf);
4730             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4731         }
4732
4733         foreach my $ext (qw(mon qmp pid vnc qga)) {
4734             unlink "/var/run/qemu-server/${vmid}.$ext";
4735         }
4736
4737         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4738     };
4739     warn $@ if $@; # avoid errors - just warn
4740 }
4741
4742 # Note: use $nockeck to skip tests if VM configuration file exists.
4743 # We need that when migration VMs to other nodes (files already moved)
4744 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4745 sub vm_stop {
4746     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4747
4748     $force = 1 if !defined($force) && !$shutdown;
4749
4750     if ($migratedfrom){
4751         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4752         kill 15, $pid if $pid;
4753         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4754         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4755         return;
4756     }
4757
4758     PVE::QemuConfig->lock_config($vmid, sub {
4759
4760         my $pid = check_running($vmid, $nocheck);
4761         return if !$pid;
4762
4763         my $conf;
4764         if (!$nocheck) {
4765             $conf = PVE::QemuConfig->load_config($vmid);
4766             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4767             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4768                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4769                 $timeout = $opts->{down} if $opts->{down};
4770             }
4771         }
4772
4773         $timeout = 60 if !defined($timeout);
4774
4775         eval {
4776             if ($shutdown) {
4777                 if (defined($conf) && $conf->{agent}) {
4778                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4779                 } else {
4780                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4781                 }
4782             } else {
4783                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4784             }
4785         };
4786         my $err = $@;
4787
4788         if (!$err) {
4789             my $count = 0;
4790             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4791                 $count++;
4792                 sleep 1;
4793             }
4794
4795             if ($count >= $timeout) {
4796                 if ($force) {
4797                     warn "VM still running - terminating now with SIGTERM\n";
4798                     kill 15, $pid;
4799                 } else {
4800                     die "VM quit/powerdown failed - got timeout\n";
4801                 }
4802             } else {
4803                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4804                 return;
4805             }
4806         } else {
4807             if ($force) {
4808                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4809                 kill 15, $pid;
4810             } else {
4811                 die "VM quit/powerdown failed\n";
4812             }
4813         }
4814
4815         # wait again
4816         $timeout = 10;
4817
4818         my $count = 0;
4819         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4820             $count++;
4821             sleep 1;
4822         }
4823
4824         if ($count >= $timeout) {
4825             warn "VM still running - terminating now with SIGKILL\n";
4826             kill 9, $pid;
4827             sleep 1;
4828         }
4829
4830         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4831    });
4832 }
4833
4834 sub vm_suspend {
4835     my ($vmid, $skiplock) = @_;
4836
4837     PVE::QemuConfig->lock_config($vmid, sub {
4838
4839         my $conf = PVE::QemuConfig->load_config($vmid);
4840
4841         PVE::QemuConfig->check_lock($conf)
4842             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4843
4844         vm_mon_cmd($vmid, "stop");
4845     });
4846 }
4847
4848 sub vm_resume {
4849     my ($vmid, $skiplock, $nocheck) = @_;
4850
4851     PVE::QemuConfig->lock_config($vmid, sub {
4852
4853         if (!$nocheck) {
4854
4855             my $conf = PVE::QemuConfig->load_config($vmid);
4856
4857             PVE::QemuConfig->check_lock($conf)
4858                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4859
4860             vm_mon_cmd($vmid, "cont");
4861
4862         } else {
4863             vm_mon_cmd_nocheck($vmid, "cont");
4864         }
4865     });
4866 }
4867
4868 sub vm_sendkey {
4869     my ($vmid, $skiplock, $key) = @_;
4870
4871     PVE::QemuConfig->lock_config($vmid, sub {
4872
4873         my $conf = PVE::QemuConfig->load_config($vmid);
4874
4875         # there is no qmp command, so we use the human monitor command
4876         vm_human_monitor_command($vmid, "sendkey $key");
4877     });
4878 }
4879
4880 sub vm_destroy {
4881     my ($storecfg, $vmid, $skiplock) = @_;
4882
4883     PVE::QemuConfig->lock_config($vmid, sub {
4884
4885         my $conf = PVE::QemuConfig->load_config($vmid);
4886
4887         if (!check_running($vmid)) {
4888             destroy_vm($storecfg, $vmid, undef, $skiplock);
4889         } else {
4890             die "VM $vmid is running - destroy failed\n";
4891         }
4892     });
4893 }
4894
4895 # pci helpers
4896
4897 sub file_write {
4898     my ($filename, $buf) = @_;
4899
4900     my $fh = IO::File->new($filename, "w");
4901     return undef if !$fh;
4902
4903     my $res = print $fh $buf;
4904
4905     $fh->close();
4906
4907     return $res;
4908 }
4909
4910 sub pci_device_info {
4911     my ($name) = @_;
4912
4913     my $res;
4914
4915     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4916     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4917
4918     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4919     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4920
4921     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4922     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4923
4924     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4925     return undef if !defined($product) || $product !~ s/^0x//;
4926
4927     $res = {
4928         name => $name,
4929         vendor => $vendor,
4930         product => $product,
4931         domain => $domain,
4932         bus => $bus,
4933         slot => $slot,
4934         func => $func,
4935         irq => $irq,
4936         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4937     };
4938
4939     return $res;
4940 }
4941
4942 sub pci_dev_reset {
4943     my ($dev) = @_;
4944
4945     my $name = $dev->{name};
4946
4947     my $fn = "$pcisysfs/devices/$name/reset";
4948
4949     return file_write($fn, "1");
4950 }
4951
4952 sub pci_dev_bind_to_vfio {
4953     my ($dev) = @_;
4954
4955     my $name = $dev->{name};
4956
4957     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4958
4959     if (!-d $vfio_basedir) {
4960         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4961     }
4962     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4963
4964     my $testdir = "$vfio_basedir/$name";
4965     return 1 if -d $testdir;
4966
4967     my $data = "$dev->{vendor} $dev->{product}";
4968     return undef if !file_write("$vfio_basedir/new_id", $data);
4969
4970     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4971     if (!file_write($fn, $name)) {
4972         return undef if -f $fn;
4973     }
4974
4975     $fn = "$vfio_basedir/bind";
4976     if (! -d $testdir) {
4977         return undef if !file_write($fn, $name);
4978     }
4979
4980     return -d $testdir;
4981 }
4982
4983 sub pci_dev_group_bind_to_vfio {
4984     my ($pciid) = @_;
4985
4986     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4987
4988     if (!-d $vfio_basedir) {
4989         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4990     }
4991     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4992
4993     # get IOMMU group devices
4994     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4995       my @devs = grep /^0000:/, readdir($D);
4996     closedir($D);
4997
4998     foreach my $pciid (@devs) {
4999         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5000
5001         # pci bridges, switches or root ports are not supported
5002         # they have a pci_bus subdirectory so skip them
5003         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5004
5005         my $info = pci_device_info($1);
5006         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5007     }
5008
5009     return 1;
5010 }
5011
5012 # vzdump restore implementaion
5013
5014 sub tar_archive_read_firstfile {
5015     my $archive = shift;
5016
5017     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5018
5019     # try to detect archive type first
5020     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5021         die "unable to open file '$archive'\n";
5022     my $firstfile = <$fh>;
5023     kill 15, $pid;
5024     close $fh;
5025
5026     die "ERROR: archive contaions no data\n" if !$firstfile;
5027     chomp $firstfile;
5028
5029     return $firstfile;
5030 }
5031
5032 sub tar_restore_cleanup {
5033     my ($storecfg, $statfile) = @_;
5034
5035     print STDERR "starting cleanup\n";
5036
5037     if (my $fd = IO::File->new($statfile, "r")) {
5038         while (defined(my $line = <$fd>)) {
5039             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5040                 my $volid = $2;
5041                 eval {
5042                     if ($volid =~ m|^/|) {
5043                         unlink $volid || die 'unlink failed\n';
5044                     } else {
5045                         PVE::Storage::vdisk_free($storecfg, $volid);
5046                     }
5047                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5048                 };
5049                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5050             } else {
5051                 print STDERR "unable to parse line in statfile - $line";
5052             }
5053         }
5054         $fd->close();
5055     }
5056 }
5057
5058 sub restore_archive {
5059     my ($archive, $vmid, $user, $opts) = @_;
5060
5061     my $format = $opts->{format};
5062     my $comp;
5063
5064     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5065         $format = 'tar' if !$format;
5066         $comp = 'gzip';
5067     } elsif ($archive =~ m/\.tar$/) {
5068         $format = 'tar' if !$format;
5069     } elsif ($archive =~ m/.tar.lzo$/) {
5070         $format = 'tar' if !$format;
5071         $comp = 'lzop';
5072     } elsif ($archive =~ m/\.vma$/) {
5073         $format = 'vma' if !$format;
5074     } elsif ($archive =~ m/\.vma\.gz$/) {
5075         $format = 'vma' if !$format;
5076         $comp = 'gzip';
5077     } elsif ($archive =~ m/\.vma\.lzo$/) {
5078         $format = 'vma' if !$format;
5079         $comp = 'lzop';
5080     } else {
5081         $format = 'vma' if !$format; # default
5082     }
5083
5084     # try to detect archive format
5085     if ($format eq 'tar') {
5086         return restore_tar_archive($archive, $vmid, $user, $opts);
5087     } else {
5088         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5089     }
5090 }
5091
5092 sub restore_update_config_line {
5093     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5094
5095     return if $line =~ m/^\#qmdump\#/;
5096     return if $line =~ m/^\#vzdump\#/;
5097     return if $line =~ m/^lock:/;
5098     return if $line =~ m/^unused\d+:/;
5099     return if $line =~ m/^parent:/;
5100     return if $line =~ m/^template:/; # restored VM is never a template
5101
5102     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5103     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5104         # try to convert old 1.X settings
5105         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5106         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5107             my ($model, $macaddr) = split(/\=/, $devconfig);
5108             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5109             my $net = {
5110                 model => $model,
5111                 bridge => "vmbr$ind",
5112                 macaddr => $macaddr,
5113             };
5114             my $netstr = print_net($net);
5115
5116             print $outfd "net$cookie->{netcount}: $netstr\n";
5117             $cookie->{netcount}++;
5118         }
5119     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5120         my ($id, $netstr) = ($1, $2);
5121         my $net = parse_net($netstr);
5122         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5123         $netstr = print_net($net);
5124         print $outfd "$id: $netstr\n";
5125     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5126         my $virtdev = $1;
5127         my $value = $3;
5128         my $di = parse_drive($virtdev, $value);
5129         if (defined($di->{backup}) && !$di->{backup}) {
5130             print $outfd "#$line";
5131         } elsif ($map->{$virtdev}) {
5132             delete $di->{format}; # format can change on restore
5133             $di->{file} = $map->{$virtdev};
5134             $value = print_drive($vmid, $di);
5135             print $outfd "$virtdev: $value\n";
5136         } else {
5137             print $outfd $line;
5138         }
5139     } else {
5140         print $outfd $line;
5141     }
5142 }
5143
5144 sub scan_volids {
5145     my ($cfg, $vmid) = @_;
5146
5147     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5148
5149     my $volid_hash = {};
5150     foreach my $storeid (keys %$info) {
5151         foreach my $item (@{$info->{$storeid}}) {
5152             next if !($item->{volid} && $item->{size});
5153             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5154             $volid_hash->{$item->{volid}} = $item;
5155         }
5156     }
5157
5158     return $volid_hash;
5159 }
5160
5161 sub is_volume_in_use {
5162     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5163
5164     my $path = PVE::Storage::path($storecfg, $volid);
5165
5166     my $scan_config = sub {
5167         my ($cref, $snapname) = @_;
5168
5169         foreach my $key (keys %$cref) {
5170             my $value = $cref->{$key};
5171             if (is_valid_drivename($key)) {
5172                 next if $skip_drive && $key eq $skip_drive;
5173                 my $drive = parse_drive($key, $value);
5174                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5175                 return 1 if $volid eq $drive->{file};
5176                 if ($drive->{file} =~ m!^/!) {
5177                     return 1 if $drive->{file} eq $path;
5178                 } else {
5179                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5180                     next if !$storeid;
5181                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5182                     next if !$scfg;
5183                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5184                 }
5185             }
5186         }
5187
5188         return 0;
5189     };
5190
5191     return 1 if &$scan_config($conf);
5192
5193     undef $skip_drive;
5194
5195     foreach my $snapname (keys %{$conf->{snapshots}}) {
5196         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5197     }
5198
5199     return 0;
5200 }
5201
5202 sub update_disksize {
5203     my ($vmid, $conf, $volid_hash) = @_;
5204
5205     my $changes;
5206
5207     my $used = {};
5208
5209     # Note: it is allowed to define multiple storages with same path (alias), so
5210     # we need to check both 'volid' and real 'path' (two different volid can point
5211     # to the same path).
5212
5213     my $usedpath = {};
5214
5215     # update size info
5216     foreach my $opt (keys %$conf) {
5217         if (is_valid_drivename($opt)) {
5218             my $drive = parse_drive($opt, $conf->{$opt});
5219             my $volid = $drive->{file};
5220             next if !$volid;
5221
5222             $used->{$volid} = 1;
5223             if ($volid_hash->{$volid} &&
5224                 (my $path = $volid_hash->{$volid}->{path})) {
5225                 $usedpath->{$path} = 1;
5226             }
5227
5228             next if drive_is_cdrom($drive);
5229             next if !$volid_hash->{$volid};
5230
5231             $drive->{size} = $volid_hash->{$volid}->{size};
5232             my $new = print_drive($vmid, $drive);
5233             if ($new ne $conf->{$opt}) {
5234                 $changes = 1;
5235                 $conf->{$opt} = $new;
5236             }
5237         }
5238     }
5239
5240     # remove 'unusedX' entry if volume is used
5241     foreach my $opt (keys %$conf) {
5242         next if $opt !~ m/^unused\d+$/;
5243         my $volid = $conf->{$opt};
5244         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5245         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5246             $changes = 1;
5247             delete $conf->{$opt};
5248         }
5249     }
5250
5251     foreach my $volid (sort keys %$volid_hash) {
5252         next if $volid =~ m/vm-$vmid-state-/;
5253         next if $used->{$volid};
5254         my $path = $volid_hash->{$volid}->{path};
5255         next if !$path; # just to be sure
5256         next if $usedpath->{$path};
5257         $changes = 1;
5258         PVE::QemuConfig->add_unused_volume($conf, $volid);
5259         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5260     }
5261
5262     return $changes;
5263 }
5264
5265 sub rescan {
5266     my ($vmid, $nolock) = @_;
5267
5268     my $cfg = PVE::Storage::config();
5269
5270     my $volid_hash = scan_volids($cfg, $vmid);
5271
5272     my $updatefn =  sub {
5273         my ($vmid) = @_;
5274
5275         my $conf = PVE::QemuConfig->load_config($vmid);
5276
5277         PVE::QemuConfig->check_lock($conf);
5278
5279         my $vm_volids = {};
5280         foreach my $volid (keys %$volid_hash) {
5281             my $info = $volid_hash->{$volid};
5282             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5283         }
5284
5285         my $changes = update_disksize($vmid, $conf, $vm_volids);
5286
5287         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5288     };
5289
5290     if (defined($vmid)) {
5291         if ($nolock) {
5292             &$updatefn($vmid);
5293         } else {
5294             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5295         }
5296     } else {
5297         my $vmlist = config_list();
5298         foreach my $vmid (keys %$vmlist) {
5299             if ($nolock) {
5300                 &$updatefn($vmid);
5301             } else {
5302                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5303             }
5304         }
5305     }
5306 }
5307
5308 sub restore_vma_archive {
5309     my ($archive, $vmid, $user, $opts, $comp) = @_;
5310
5311     my $input = $archive eq '-' ? "<&STDIN" : undef;
5312     my $readfrom = $archive;
5313
5314     my $uncomp = '';
5315     if ($comp) {
5316         $readfrom = '-';
5317         my $qarchive = PVE::Tools::shellquote($archive);
5318         if ($comp eq 'gzip') {
5319             $uncomp = "zcat $qarchive|";
5320         } elsif ($comp eq 'lzop') {
5321             $uncomp = "lzop -d -c $qarchive|";
5322         } else {
5323             die "unknown compression method '$comp'\n";
5324         }
5325
5326     }
5327
5328     my $tmpdir = "/var/tmp/vzdumptmp$$";
5329     rmtree $tmpdir;
5330
5331     # disable interrupts (always do cleanups)
5332     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5333         warn "got interrupt - ignored\n";
5334     };
5335
5336     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5337     POSIX::mkfifo($mapfifo, 0600);
5338     my $fifofh;
5339
5340     my $openfifo = sub {
5341         open($fifofh, '>', $mapfifo) || die $!;
5342     };
5343
5344     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5345
5346     my $oldtimeout;
5347     my $timeout = 5;
5348
5349     my $devinfo = {};
5350
5351     my $rpcenv = PVE::RPCEnvironment::get();
5352
5353     my $conffile = PVE::QemuConfig->config_file($vmid);
5354     my $tmpfn = "$conffile.$$.tmp";
5355
5356     # Note: $oldconf is undef if VM does not exists
5357     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5358     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5359
5360     my $print_devmap = sub {
5361         my $virtdev_hash = {};
5362
5363         my $cfgfn = "$tmpdir/qemu-server.conf";
5364
5365         # we can read the config - that is already extracted
5366         my $fh = IO::File->new($cfgfn, "r") ||
5367             "unable to read qemu-server.conf - $!\n";
5368
5369         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5370         if (-f $fwcfgfn) {
5371             my $pve_firewall_dir = '/etc/pve/firewall';
5372             mkdir $pve_firewall_dir; # make sure the dir exists
5373             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5374         }
5375
5376         while (defined(my $line = <$fh>)) {
5377             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5378                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5379                 die "archive does not contain data for drive '$virtdev'\n"
5380                     if !$devinfo->{$devname};
5381                 if (defined($opts->{storage})) {
5382                     $storeid = $opts->{storage} || 'local';
5383                 } elsif (!$storeid) {
5384                     $storeid = 'local';
5385                 }
5386                 $format = 'raw' if !$format;
5387                 $devinfo->{$devname}->{devname} = $devname;
5388                 $devinfo->{$devname}->{virtdev} = $virtdev;
5389                 $devinfo->{$devname}->{format} = $format;
5390                 $devinfo->{$devname}->{storeid} = $storeid;
5391
5392                 # check permission on storage
5393                 my $pool = $opts->{pool}; # todo: do we need that?
5394                 if ($user ne 'root@pam') {
5395                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5396                 }
5397
5398                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5399             }
5400         }
5401
5402         foreach my $devname (keys %$devinfo) {
5403             die "found no device mapping information for device '$devname'\n"
5404                 if !$devinfo->{$devname}->{virtdev};
5405         }
5406
5407         my $cfg = PVE::Storage::config();
5408
5409         # create empty/temp config
5410         if ($oldconf) {
5411             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5412             foreach_drive($oldconf, sub {
5413                 my ($ds, $drive) = @_;
5414
5415                 return if drive_is_cdrom($drive);
5416
5417                 my $volid = $drive->{file};
5418
5419                 return if !$volid || $volid =~ m|^/|;
5420
5421                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5422                 return if !$path || !$owner || ($owner != $vmid);
5423
5424                 # Note: only delete disk we want to restore
5425                 # other volumes will become unused
5426                 if ($virtdev_hash->{$ds}) {
5427                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5428                     if (my $err = $@) {
5429                         warn $err;
5430                     }
5431                 }
5432             });
5433
5434             # delete vmstate files
5435             # since after the restore we have no snapshots anymore
5436             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5437                 my $snap = $oldconf->{snapshots}->{$snapname};
5438                 if ($snap->{vmstate}) {
5439                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5440                     if (my $err = $@) {
5441                         warn $err;
5442                     }
5443                 }
5444             }
5445         }
5446
5447         my $map = {};
5448         foreach my $virtdev (sort keys %$virtdev_hash) {
5449             my $d = $virtdev_hash->{$virtdev};
5450             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5451             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5452
5453             # test if requested format is supported
5454             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5455             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5456             $d->{format} = $defFormat if !$supported;
5457
5458             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5459                                                   $d->{format}, undef, $alloc_size);
5460             print STDERR "new volume ID is '$volid'\n";
5461             $d->{volid} = $volid;
5462             my $path = PVE::Storage::path($cfg, $volid);
5463
5464             PVE::Storage::activate_volumes($cfg,[$volid]);
5465
5466             my $write_zeros = 1;
5467             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5468                 $write_zeros = 0;
5469             }
5470
5471             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5472
5473             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5474             $map->{$virtdev} = $volid;
5475         }
5476
5477         $fh->seek(0, 0) || die "seek failed - $!\n";
5478
5479         my $outfd = new IO::File ($tmpfn, "w") ||
5480             die "unable to write config for VM $vmid\n";
5481
5482         my $cookie = { netcount => 0 };
5483         while (defined(my $line = <$fh>)) {
5484             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5485         }
5486
5487         $fh->close();
5488         $outfd->close();
5489     };
5490
5491     eval {
5492         # enable interrupts
5493         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5494             die "interrupted by signal\n";
5495         };
5496         local $SIG{ALRM} = sub { die "got timeout\n"; };
5497
5498         $oldtimeout = alarm($timeout);
5499
5500         my $parser = sub {
5501             my $line = shift;
5502
5503             print "$line\n";
5504
5505             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5506                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5507                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5508             } elsif ($line =~ m/^CTIME: /) {
5509                 # we correctly received the vma config, so we can disable
5510                 # the timeout now for disk allocation (set to 10 minutes, so
5511                 # that we always timeout if something goes wrong)
5512                 alarm(600);
5513                 &$print_devmap();
5514                 print $fifofh "done\n";
5515                 my $tmp = $oldtimeout || 0;
5516                 $oldtimeout = undef;
5517                 alarm($tmp);
5518                 close($fifofh);
5519             }
5520         };
5521
5522         print "restore vma archive: $cmd\n";
5523         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5524     };
5525     my $err = $@;
5526
5527     alarm($oldtimeout) if $oldtimeout;
5528
5529     my $vollist = [];
5530     foreach my $devname (keys %$devinfo) {
5531         my $volid = $devinfo->{$devname}->{volid};
5532         push @$vollist, $volid if $volid;
5533     }
5534
5535     my $cfg = PVE::Storage::config();
5536     PVE::Storage::deactivate_volumes($cfg, $vollist);
5537
5538     unlink $mapfifo;
5539
5540     if ($err) {
5541         rmtree $tmpdir;
5542         unlink $tmpfn;
5543
5544         foreach my $devname (keys %$devinfo) {
5545             my $volid = $devinfo->{$devname}->{volid};
5546             next if !$volid;
5547             eval {
5548                 if ($volid =~ m|^/|) {
5549                     unlink $volid || die 'unlink failed\n';
5550                 } else {
5551                     PVE::Storage::vdisk_free($cfg, $volid);
5552                 }
5553                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5554             };
5555             print STDERR "unable to cleanup '$volid' - $@" if $@;
5556         }
5557         die $err;
5558     }
5559
5560     rmtree $tmpdir;
5561
5562     rename($tmpfn, $conffile) ||
5563         die "unable to commit configuration file '$conffile'\n";
5564
5565     PVE::Cluster::cfs_update(); # make sure we read new file
5566
5567     eval { rescan($vmid, 1); };
5568     warn $@ if $@;
5569 }
5570
5571 sub restore_tar_archive {
5572     my ($archive, $vmid, $user, $opts) = @_;
5573
5574     if ($archive ne '-') {
5575         my $firstfile = tar_archive_read_firstfile($archive);
5576         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5577             if $firstfile ne 'qemu-server.conf';
5578     }
5579
5580     my $storecfg = PVE::Storage::config();
5581
5582     # destroy existing data - keep empty config
5583     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5584     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5585
5586     my $tocmd = "/usr/lib/qemu-server/qmextract";
5587
5588     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5589     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5590     $tocmd .= ' --prealloc' if $opts->{prealloc};
5591     $tocmd .= ' --info' if $opts->{info};
5592
5593     # tar option "xf" does not autodetect compression when read from STDIN,
5594     # so we pipe to zcat
5595     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5596         PVE::Tools::shellquote("--to-command=$tocmd");
5597
5598     my $tmpdir = "/var/tmp/vzdumptmp$$";
5599     mkpath $tmpdir;
5600
5601     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5602     local $ENV{VZDUMP_VMID} = $vmid;
5603     local $ENV{VZDUMP_USER} = $user;
5604
5605     my $conffile = PVE::QemuConfig->config_file($vmid);
5606     my $tmpfn = "$conffile.$$.tmp";
5607
5608     # disable interrupts (always do cleanups)
5609     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5610         print STDERR "got interrupt - ignored\n";
5611     };
5612
5613     eval {
5614         # enable interrupts
5615         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5616             die "interrupted by signal\n";
5617         };
5618
5619         if ($archive eq '-') {
5620             print "extracting archive from STDIN\n";
5621             run_command($cmd, input => "<&STDIN");
5622         } else {
5623             print "extracting archive '$archive'\n";
5624             run_command($cmd);
5625         }
5626
5627         return if $opts->{info};
5628
5629         # read new mapping
5630         my $map = {};
5631         my $statfile = "$tmpdir/qmrestore.stat";
5632         if (my $fd = IO::File->new($statfile, "r")) {
5633             while (defined (my $line = <$fd>)) {
5634                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5635                     $map->{$1} = $2 if $1;
5636                 } else {
5637                     print STDERR "unable to parse line in statfile - $line\n";
5638                 }
5639             }
5640             $fd->close();
5641         }
5642
5643         my $confsrc = "$tmpdir/qemu-server.conf";
5644
5645         my $srcfd = new IO::File($confsrc, "r") ||
5646             die "unable to open file '$confsrc'\n";
5647
5648         my $outfd = new IO::File ($tmpfn, "w") ||
5649             die "unable to write config for VM $vmid\n";
5650
5651         my $cookie = { netcount => 0 };
5652         while (defined (my $line = <$srcfd>)) {
5653             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5654         }
5655
5656         $srcfd->close();
5657         $outfd->close();
5658     };
5659     my $err = $@;
5660
5661     if ($err) {
5662
5663         unlink $tmpfn;
5664
5665         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5666
5667         die $err;
5668     }
5669
5670     rmtree $tmpdir;
5671
5672     rename $tmpfn, $conffile ||
5673         die "unable to commit configuration file '$conffile'\n";
5674
5675     PVE::Cluster::cfs_update(); # make sure we read new file
5676
5677     eval { rescan($vmid, 1); };
5678     warn $@ if $@;
5679 };
5680
5681 sub foreach_writable_storage {
5682     my ($conf, $func) = @_;
5683
5684     my $sidhash = {};
5685
5686     foreach my $ds (keys %$conf) {
5687         next if !is_valid_drivename($ds);
5688
5689         my $drive = parse_drive($ds, $conf->{$ds});
5690         next if !$drive;
5691         next if drive_is_cdrom($drive);
5692
5693         my $volid = $drive->{file};
5694
5695         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5696         $sidhash->{$sid} = $sid if $sid;
5697     }
5698
5699     foreach my $sid (sort keys %$sidhash) {
5700         &$func($sid);
5701     }
5702 }
5703
5704 sub do_snapshots_with_qemu {
5705     my ($storecfg, $volid) = @_;
5706
5707     my $storage_name = PVE::Storage::parse_volume_id($volid);
5708
5709     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5710         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5711         return 1;
5712     }
5713
5714     if ($volid =~ m/\.(qcow2|qed)$/){
5715         return 1;
5716     }
5717
5718     return undef;
5719 }
5720
5721 sub qga_check_running {
5722     my ($vmid) = @_;
5723
5724     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5725     if ($@) {
5726         warn "Qemu Guest Agent are not running - $@";
5727         return 0;
5728     }
5729     return 1;
5730 }
5731
5732 sub template_create {
5733     my ($vmid, $conf, $disk) = @_;
5734
5735     my $storecfg = PVE::Storage::config();
5736
5737     foreach_drive($conf, sub {
5738         my ($ds, $drive) = @_;
5739
5740         return if drive_is_cdrom($drive);
5741         return if $disk && $ds ne $disk;
5742
5743         my $volid = $drive->{file};
5744         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5745
5746         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5747         $drive->{file} = $voliddst;
5748         $conf->{$ds} = print_drive($vmid, $drive);
5749         PVE::QemuConfig->write_config($vmid, $conf);
5750     });
5751 }
5752
5753 sub qemu_img_convert {
5754     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5755
5756     my $storecfg = PVE::Storage::config();
5757     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5758     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5759
5760     if ($src_storeid && $dst_storeid) {
5761
5762         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5763
5764         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5765         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5766
5767         my $src_format = qemu_img_format($src_scfg, $src_volname);
5768         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5769
5770         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5771         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5772
5773         my $cmd = [];
5774         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5775         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5776         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5777         if ($is_zero_initialized) {
5778             push @$cmd, "zeroinit:$dst_path";
5779         } else {
5780             push @$cmd, $dst_path;
5781         }
5782
5783         my $parser = sub {
5784             my $line = shift;
5785             if($line =~ m/\((\S+)\/100\%\)/){
5786                 my $percent = $1;
5787                 my $transferred = int($size * $percent / 100);
5788                 my $remaining = $size - $transferred;
5789
5790                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5791             }
5792
5793         };
5794
5795         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5796         my $err = $@;
5797         die "copy failed: $err" if $err;
5798     }
5799 }
5800
5801 sub qemu_img_format {
5802     my ($scfg, $volname) = @_;
5803
5804     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5805         return $1;
5806     } else {
5807         return "raw";
5808     }
5809 }
5810
5811 sub qemu_drive_mirror {
5812     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5813
5814     my $storecfg = PVE::Storage::config();
5815     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5816
5817     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5818
5819     my $format = qemu_img_format($dst_scfg, $dst_volname);
5820
5821     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5822
5823     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5824
5825     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5826     $opts->{format} = $format if $format;
5827
5828     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5829
5830     my $finish_job = sub {
5831         while (1) {
5832             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5833             my $stat = @$stats[0];
5834             last if !$stat;
5835             sleep 1;
5836         }
5837     };
5838
5839     eval {
5840     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5841         while (1) {
5842             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5843             my $stat = @$stats[0];
5844             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5845             die "error job is not mirroring" if $stat->{type} ne "mirror";
5846
5847             my $busy = $stat->{busy};
5848             my $ready = $stat->{ready};
5849
5850             if (my $total = $stat->{len}) {
5851                 my $transferred = $stat->{offset} || 0;
5852                 my $remaining = $total - $transferred;
5853                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5854
5855                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5856             }
5857
5858
5859             if ($stat->{ready} eq 'true') {
5860
5861                 last if $vmiddst != $vmid;
5862
5863                 # try to switch the disk if source and destination are on the same guest
5864                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5865                 if (!$@) {
5866                     &$finish_job();
5867                     last;
5868                 }
5869                 die $@ if $@ !~ m/cannot be completed/;
5870             }
5871             sleep 1;
5872         }
5873
5874
5875     };
5876     my $err = $@;
5877
5878     my $cancel_job = sub {
5879         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5880         &$finish_job();
5881     };
5882
5883     if ($err) {
5884         eval { &$cancel_job(); };
5885         die "mirroring error: $err";
5886     }
5887
5888     if ($vmiddst != $vmid) {
5889         # if we clone a disk for a new target vm, we don't switch the disk
5890         &$cancel_job(); # so we call block-job-cancel
5891     }
5892 }
5893
5894 sub clone_disk {
5895     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5896         $newvmid, $storage, $format, $full, $newvollist) = @_;
5897
5898     my $newvolid;
5899
5900     if (!$full) {
5901         print "create linked clone of drive $drivename ($drive->{file})\n";
5902         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5903         push @$newvollist, $newvolid;
5904     } else {
5905         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5906         $storeid = $storage if $storage;
5907
5908         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5909         if (!$format) {
5910             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5911             $format = qemu_img_format($scfg, $volname);
5912         }
5913
5914         # test if requested format is supported - else use default
5915         my $supported = grep { $_ eq $format } @$validFormats;
5916         $format = $defFormat if !$supported;
5917
5918         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5919
5920         print "create full clone of drive $drivename ($drive->{file})\n";
5921         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5922         push @$newvollist, $newvolid;
5923
5924         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
5925
5926         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5927         if (!$running || $snapname) {
5928             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5929         } else {
5930
5931             my $kvmver = get_running_qemu_version ($vmid);
5932             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
5933                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
5934                     if $drive->{iothread};
5935             }
5936
5937             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5938         }
5939     }
5940
5941     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5942
5943     my $disk = $drive;
5944     $disk->{format} = undef;
5945     $disk->{file} = $newvolid;
5946     $disk->{size} = $size;
5947
5948     return $disk;
5949 }
5950
5951 # this only works if VM is running
5952 sub get_current_qemu_machine {
5953     my ($vmid) = @_;
5954
5955     my $cmd = { execute => 'query-machines', arguments => {} };
5956     my $res = vm_qmp_command($vmid, $cmd);
5957
5958     my ($current, $default);
5959     foreach my $e (@$res) {
5960         $default = $e->{name} if $e->{'is-default'};
5961         $current = $e->{name} if $e->{'is-current'};
5962     }
5963
5964     # fallback to the default machine if current is not supported by qemu
5965     return $current || $default || 'pc';
5966 }
5967
5968 sub get_running_qemu_version {
5969     my ($vmid) = @_;
5970     my $cmd = { execute => 'query-version', arguments => {} };
5971     my $res = vm_qmp_command($vmid, $cmd);
5972     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
5973 }
5974
5975 sub qemu_machine_feature_enabled {
5976     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5977
5978     my $current_major;
5979     my $current_minor;
5980
5981     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5982
5983         $current_major = $3;
5984         $current_minor = $4;
5985
5986     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5987
5988         $current_major = $1;
5989         $current_minor = $2;
5990     }
5991
5992     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5993
5994
5995 }
5996
5997 sub qemu_machine_pxe {
5998     my ($vmid, $conf, $machine) = @_;
5999
6000     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6001
6002     foreach my $opt (keys %$conf) {
6003         next if $opt !~ m/^net(\d+)$/;
6004         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6005         next if !$net;
6006         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6007         return $machine.".pxe" if $romfile =~ m/pxe/;
6008         last;
6009     }
6010
6011     return $machine;
6012 }
6013
6014 sub qemu_use_old_bios_files {
6015     my ($machine_type) = @_;
6016
6017     return if !$machine_type;
6018
6019     my $use_old_bios_files = undef;
6020
6021     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6022         $machine_type = $1;
6023         $use_old_bios_files = 1;
6024     } else {
6025         my $kvmver = kvm_user_version();
6026         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6027         # load new efi bios files on migration. So this hack is required to allow
6028         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6029         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6030         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6031     }
6032
6033     return ($use_old_bios_files, $machine_type);
6034 }
6035
6036 sub lspci {
6037
6038     my $devices = {};
6039
6040     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6041             my (undef, $id, $function) = @_;
6042             my $res = { id => $id, function => $function};
6043             push @{$devices->{$id}}, $res;
6044     });
6045
6046     return $devices;
6047 }
6048
6049 sub vm_iothreads_list {
6050     my ($vmid) = @_;
6051
6052     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6053
6054     my $iothreads = {};
6055     foreach my $iothread (@$res) {
6056         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6057     }
6058
6059     return $iothreads;
6060 }
6061
6062 sub scsihw_infos {
6063     my ($conf, $drive) = @_;
6064
6065     my $maxdev = 0;
6066
6067     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6068         $maxdev = 7;
6069     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6070         $maxdev = 1;
6071     } else {
6072         $maxdev = 256;
6073     }
6074
6075     my $controller = int($drive->{index} / $maxdev);
6076     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6077
6078     return ($maxdev, $controller, $controller_prefix);
6079 }
6080
6081 # bash completion helper
6082
6083 sub complete_backup_archives {
6084     my ($cmdname, $pname, $cvalue) = @_;
6085
6086     my $cfg = PVE::Storage::config();
6087
6088     my $storeid;
6089
6090     if ($cvalue =~ m/^([^:]+):/) {
6091         $storeid = $1;
6092     }
6093
6094     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6095
6096     my $res = [];
6097     foreach my $id (keys %$data) {
6098         foreach my $item (@{$data->{$id}}) {
6099             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6100             push @$res, $item->{volid} if defined($item->{volid});
6101         }
6102     }
6103
6104     return $res;
6105 }
6106
6107 my $complete_vmid_full = sub {
6108     my ($running) = @_;
6109
6110     my $idlist = vmstatus();
6111
6112     my $res = [];
6113
6114     foreach my $id (keys %$idlist) {
6115         my $d = $idlist->{$id};
6116         if (defined($running)) {
6117             next if $d->{template};
6118             next if $running && $d->{status} ne 'running';
6119             next if !$running && $d->{status} eq 'running';
6120         }
6121         push @$res, $id;
6122
6123     }
6124     return $res;
6125 };
6126
6127 sub complete_vmid {
6128     return &$complete_vmid_full();
6129 }
6130
6131 sub complete_vmid_stopped {
6132     return &$complete_vmid_full(0);
6133 }
6134
6135 sub complete_vmid_running {
6136     return &$complete_vmid_full(1);
6137 }
6138
6139 sub complete_storage {
6140
6141     my $cfg = PVE::Storage::config();
6142     my $ids = $cfg->{ids};
6143
6144     my $res = [];
6145     foreach my $sid (keys %$ids) {
6146         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6147         next if !$ids->{$sid}->{content}->{images};
6148         push @$res, $sid;
6149     }
6150
6151     return $res;
6152 }
6153
6154 1;